diff --git a/.gitignore b/.gitignore index e3ccf50508f..0c976a1a226 100644 --- a/.gitignore +++ b/.gitignore @@ -106,6 +106,13 @@ STABILIZATION_TODO.md **/coverage test-config +# Claude Code compatibility-matrix pytest artifact (CI-only output). +compat-results.json +compat-results.json.shards/ +compat-rate-limit-summary.json +# Matrix JSON produced by the daily-cron publisher (pushed to litellm-docs). +compatibility-matrix.json + # ---------- Terraform ---------- # Provider binaries + module cache — regenerated by `terraform init`. **/.terraform/ diff --git a/pyrightconfig.json b/pyrightconfig.json index 97f099d5b2c..eabfbf515c4 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,7 +1,7 @@ { "include": ["litellm"], "ignore": [], - "exclude": ["**/node_modules", "**/__pycache__", "litellm/types/utils.py", "litellm/proxy/_types.py"], + "exclude": ["**/node_modules", "**/__pycache__", "tests/e2e/claude_code", "litellm/types/utils.py", "litellm/proxy/_types.py"], "pythonVersion": "3.12", "typeCheckingMode": "strict", "enableTypeIgnoreComments": false, diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 15e35c30ef7..f0d283629b0 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -17,6 +17,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests +- `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees, and does not use the shared transport harness ## Lay the pattern down in a class diff --git a/tests/e2e/claude_code/__init__.py b/tests/e2e/claude_code/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/_basic_messaging.py b/tests/e2e/claude_code/_basic_messaging.py new file mode 100644 index 00000000000..f6b82a38f6a --- /dev/null +++ b/tests/e2e/claude_code/_basic_messaging.py @@ -0,0 +1,167 @@ +"""Shared body for the `basic_messaging_*` × compat cells. + +Every basic_messaging cell follows the same skeleton: + + 1. Read the proxy base URL + API key from env, fail-early if missing. + 2. Fan the three Claude tiers out via `run_claude_models_parallel`. + 3. Inspect each model's outcome and report one `compat_result` row per + model — `ClaudeCLIError`, non-zero exit, and empty assistant text + are all per-model fails; everything else is a per-model pass. + 4. Surface a joined failure message via `pytest.fail(...)` so the + pytest run also goes red. + +The streaming variant additionally passes `verify_streaming=True`, +which adds the `--include-partial-messages` CLI flag and asserts that +the proxy actually streamed the response (see the helper docstring for +the wire-level rationale). + +The conftest infers `(feature_id, provider)` purely from the test file +path, so each per-provider file just declares its model list and calls +`run_basic_messaging_cell(...)`. This keeps all cell logic in one place +— a future tweak to the env-missing guard or the failure-loop shape +now propagates to every cell automatically. + +The leading underscore in the filename is what keeps pytest from +collecting this module as a test file. +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +# Floor on the number of `stream_event` records (with delta payloads) +# we expect to see when the proxy actually streams. With +# `--include-partial-messages`, the CLI emits one `stream_event` per +# raw upstream SSE event — a fully-streamed response produces many +# (`message_start`, multiple `content_block_delta`s, `content_block_stop`, +# `message_delta`, `message_stop`); a proxy that buffers the upstream +# and returns a single non-streaming chunk produces 0 or 1. Floor of 2 +# is safely above the buffered case for any non-trivial reply, which +# is why the streaming cells use a "count from 1 to 5" style prompt. +MIN_STREAM_DELTA_EVENTS = 2 + + +def _count_stream_event_deltas(events: Sequence[Mapping[str, Any]]) -> int: + """Count `stream_event` records that carry an SSE event payload. + + With `--include-partial-messages`, Claude Code wraps every upstream + SSE event in a `{"type": "stream_event", "event": {...}}` record. + A buffering proxy collapses the upstream stream into a single + non-streaming response, so these records vanish. Counting them + (rather than just `len(events)`) is the wire-level signal that + "did the proxy preserve streaming?" — independent of the `system` + /`assistant`/`result` boilerplate records the CLI always emits. + """ + count = 0 + for event in events: + if event.get("type") != "stream_event": + continue + if isinstance(event.get("event"), Mapping): + count += 1 + return count + + +def run_basic_messaging_cell( + *, + compat_result, + models: Sequence[str], + prompt: str, + verify_streaming: bool = False, +) -> None: + """Run the shared `basic_messaging_*` × cell body. + + When ``verify_streaming=True``, the cell additionally asserts that + the proxy streamed the response end-to-end. The check works by + passing ``--include-partial-messages`` to the `claude` CLI, which + causes it to emit one ``stream_event`` record per raw upstream SSE + event (``message_start``, ``content_block_delta``, ``message_stop``, + etc.). A proxy that buffers the upstream stream and returns a + single non-streaming response collapses those records to zero — + so a floor of ``MIN_STREAM_DELTA_EVENTS`` ``stream_event`` records + catches the buffering regression without needing a streaming-aware + driver. + + This is the same shape of check as ``tool_use_streaming`` uses, + just keyed off the explicit partial-message flag so it works for + plain assistant replies (where the CLI would otherwise collapse a + streamed reply to a single ``assistant`` event in + ``--print --output-format stream-json`` mode). + """ + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + extra_args: Sequence[str] = ( + ("--include-partial-messages",) if verify_streaming else () + ) + + outcomes = run_claude_models_parallel( + models=models, + prompt=prompt, + base_url=base_url, + api_key=api_key, + extra_args=extra_args, + ) + + failures = [] + for model in models: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if verify_streaming: + stream_event_count = _count_stream_event_deltas(outcome.events) + if stream_event_count < MIN_STREAM_DELTA_EVENTS: + error = ( + f"[{model}] only {stream_event_count} stream_event records " + f"observed (< {MIN_STREAM_DELTA_EVENTS}); proxy likely " + f"buffered the upstream response instead of streaming it" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/_builder_unit_tests/__init__.py b/tests/e2e/claude_code/_builder_unit_tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/_builder_unit_tests/fixtures/expected_matrix.json b/tests/e2e/claude_code/_builder_unit_tests/fixtures/expected_matrix.json new file mode 100644 index 00000000000..d3aca0142dc --- /dev/null +++ b/tests/e2e/claude_code/_builder_unit_tests/fixtures/expected_matrix.json @@ -0,0 +1,38 @@ +{ + "schema_version": "1", + "generated_at": "2026-04-25T00:00:00Z", + "litellm_version": "v1.83.0-stable", + "claude_code_version": "2.1.120", + "providers": [ + "anthropic", + "bedrock_invoke" + ], + "features": [ + { + "id": "basic_messaging_non_streaming", + "name": "Basic messaging (non-streaming)", + "providers": { + "anthropic": { + "status": "pass" + }, + "bedrock_invoke": { + "status": "not_tested" + } + } + }, + { + "id": "tool_use", + "name": "Tool use", + "providers": { + "anthropic": { + "status": "fail", + "error": "[claude-sonnet-4-6] tool call dropped" + }, + "bedrock_invoke": { + "status": "not_applicable", + "reason": "tool use not yet wired up for Bedrock Invoke" + } + } + } + ] +} diff --git a/tests/e2e/claude_code/_builder_unit_tests/fixtures/manifest.yaml b/tests/e2e/claude_code/_builder_unit_tests/fixtures/manifest.yaml new file mode 100644 index 00000000000..e88bdc6ddf5 --- /dev/null +++ b/tests/e2e/claude_code/_builder_unit_tests/fixtures/manifest.yaml @@ -0,0 +1,9 @@ +schema_version: "1" +providers: + - anthropic + - bedrock_invoke +features: + - id: basic_messaging_non_streaming + name: Basic messaging (non-streaming) + - id: tool_use + name: Tool use diff --git a/tests/e2e/claude_code/_builder_unit_tests/fixtures/results.json b/tests/e2e/claude_code/_builder_unit_tests/fixtures/results.json new file mode 100644 index 00000000000..a01540c394f --- /dev/null +++ b/tests/e2e/claude_code/_builder_unit_tests/fixtures/results.json @@ -0,0 +1,41 @@ +{ + "schema_version": "1", + "results": [ + { + "feature_id": "basic_messaging_non_streaming", + "provider": "anthropic", + "nodeid": "tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py::test_basic_messaging_non_streaming_anthropic[claude-haiku-4-5]", + "result": {"status": "pass"} + }, + { + "feature_id": "basic_messaging_non_streaming", + "provider": "anthropic", + "nodeid": "tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py::test_basic_messaging_non_streaming_anthropic[claude-sonnet-4-6]", + "result": {"status": "pass"} + }, + { + "feature_id": "basic_messaging_non_streaming", + "provider": "anthropic", + "nodeid": "tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py::test_basic_messaging_non_streaming_anthropic[claude-opus-4-7]", + "result": {"status": "pass"} + }, + { + "feature_id": "tool_use", + "provider": "anthropic", + "nodeid": "tests/e2e/claude_code/tool_use/test_anthropic.py::test_x[claude-haiku-4-5]", + "result": {"status": "pass"} + }, + { + "feature_id": "tool_use", + "provider": "anthropic", + "nodeid": "tests/e2e/claude_code/tool_use/test_anthropic.py::test_x[claude-sonnet-4-6]", + "result": {"status": "fail", "error": "[claude-sonnet-4-6] tool call dropped"} + }, + { + "feature_id": "tool_use", + "provider": "bedrock_invoke", + "nodeid": "tests/e2e/claude_code/tool_use/test_bedrock_invoke.py::test_x[claude-haiku-4-5]", + "result": {"status": "not_applicable", "reason": "tool use not yet wired up for Bedrock Invoke"} + } + ] +} diff --git a/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py b/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py new file mode 100644 index 00000000000..9ddbdd29846 --- /dev/null +++ b/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py @@ -0,0 +1,479 @@ +"""Golden-file tests for the Matrix JSON Builder. + +These tests fix the published JSON schema. The builder is a pure function +from (manifest, results, metadata) → matrix dict, so we feed it a fixture +input set and compare the produced dict to a checked-in expected output. + +Any schema drift — intentional or accidental — surfaces as a diff in PR +review. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from claude_code.matrix_builder import ( + ManifestError, + ResultsError, + build_from_paths, + build_matrix, + load_manifest, + load_results, +) + +FIXTURES = Path(__file__).parent / "fixtures" + + +def test_build_matrix_matches_golden_file(tmp_path): + manifest = load_manifest(FIXTURES / "manifest.yaml") + results = load_results(FIXTURES / "results.json") + matrix = build_matrix( + manifest=manifest, + results=results, + litellm_version="v1.83.0-stable", + claude_code_version="2.1.120", + generated_at="2026-04-25T00:00:00Z", + ) + expected = json.loads((FIXTURES / "expected_matrix.json").read_text()) + assert matrix == expected + + +def test_build_matrix_pass_requires_all_models_pass(): + """Multiple results in one cell must all be pass for the cell to be pass.""" + manifest = { + "schema_version": "1", + "providers": ["anthropic"], + "features": [{"id": "f", "name": "F"}], + } + results = [ + {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, + {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, + {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, + ] + matrix = build_matrix( + manifest=manifest, + results=results, + litellm_version="v", + claude_code_version="c", + generated_at="t", + ) + assert matrix["features"][0]["providers"]["anthropic"] == {"status": "pass"} + + +def test_build_matrix_any_fail_makes_cell_fail(): + manifest = { + "schema_version": "1", + "providers": ["anthropic"], + "features": [{"id": "f", "name": "F"}], + } + results = [ + {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, + { + "feature_id": "f", + "provider": "anthropic", + "result": {"status": "fail", "error": "[claude-opus-4-7] timeout"}, + }, + {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, + ] + matrix = build_matrix( + manifest=manifest, + results=results, + litellm_version="v", + claude_code_version="c", + generated_at="t", + ) + cell = matrix["features"][0]["providers"]["anthropic"] + assert cell["status"] == "fail" + assert cell["error"] == "[claude-opus-4-7] timeout" + + +def test_build_matrix_joins_all_failure_errors_in_one_cell(): + """When multiple tiers fail for different reasons within the same cell, + every failure's error must appear in the published cell so triage + isn't reduced to a single tier's diagnostic. + """ + manifest = { + "schema_version": "1", + "providers": ["anthropic"], + "features": [{"id": "f", "name": "F"}], + } + results = [ + { + "feature_id": "f", + "provider": "anthropic", + "result": {"status": "fail", "error": "[claude-haiku-4-5] 429"}, + }, + {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, + { + "feature_id": "f", + "provider": "anthropic", + "result": {"status": "fail", "error": "[claude-opus-4-7] timeout"}, + }, + ] + matrix = build_matrix( + manifest=manifest, + results=results, + litellm_version="v", + claude_code_version="c", + generated_at="t", + ) + cell = matrix["features"][0]["providers"]["anthropic"] + assert cell["status"] == "fail" + assert "[claude-haiku-4-5] 429" in cell["error"] + assert "[claude-opus-4-7] timeout" in cell["error"] + + +def test_build_matrix_mixed_pass_and_not_tested_surfaces_pass(): + """A `not_tested` row mixed with `pass` rows must not silently demote + the cell to `not_tested` — `not_tested` is "absent data", not a + negative signal. Otherwise a partial crash mid-test, or a test that + explicitly recorded "tier didn't run", would discard real passing + results from the published cell. + """ + manifest = { + "schema_version": "1", + "providers": ["anthropic"], + "features": [{"id": "f", "name": "F"}], + } + results = [ + {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, + { + "feature_id": "f", + "provider": "anthropic", + "result": {"status": "not_tested"}, + }, + {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, + ] + matrix = build_matrix( + manifest=manifest, + results=results, + litellm_version="v", + claude_code_version="c", + generated_at="t", + ) + assert matrix["features"][0]["providers"]["anthropic"] == {"status": "pass"} + + +def test_build_matrix_all_not_tested_stays_not_tested(): + """A cell whose every row is `not_tested` (or empty) must remain + `not_tested` — the absent-data rule only drops `not_tested` rows + when there's other signal to surface. + """ + manifest = { + "schema_version": "1", + "providers": ["anthropic"], + "features": [{"id": "f", "name": "F"}], + } + results = [ + { + "feature_id": "f", + "provider": "anthropic", + "result": {"status": "not_tested"}, + }, + { + "feature_id": "f", + "provider": "anthropic", + "result": {"status": "not_tested"}, + }, + ] + matrix = build_matrix( + manifest=manifest, + results=results, + litellm_version="v", + claude_code_version="c", + generated_at="t", + ) + assert matrix["features"][0]["providers"]["anthropic"] == {"status": "not_tested"} + + +def test_build_matrix_mixed_pass_and_not_applicable_surfaces_pass(): + """A `not_applicable` row mixed with `pass` rows must surface as + `pass`, not `not_applicable`. The published cell answers "does this + feature work on this provider?"; if any tier passes, the feature + works there. Discarding passing tiers because one tier is NA would + misrepresent the cell as unsupported. + """ + manifest = { + "schema_version": "1", + "providers": ["anthropic"], + "features": [{"id": "f", "name": "F"}], + } + results = [ + {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, + { + "feature_id": "f", + "provider": "anthropic", + "result": { + "status": "not_applicable", + "reason": "haiku does not support extended thinking", + }, + }, + {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, + ] + matrix = build_matrix( + manifest=manifest, + results=results, + litellm_version="v", + claude_code_version="c", + generated_at="t", + ) + assert matrix["features"][0]["providers"]["anthropic"] == {"status": "pass"} + + +def test_build_matrix_all_not_applicable_stays_not_applicable(): + """When every observed row is `not_applicable`, the cell remains + `not_applicable` and the first row's reason carries through to the + published matrix. + """ + manifest = { + "schema_version": "1", + "providers": ["anthropic"], + "features": [{"id": "f", "name": "F"}], + } + results = [ + { + "feature_id": "f", + "provider": "anthropic", + "result": { + "status": "not_applicable", + "reason": "feature unsupported on this provider", + }, + }, + { + "feature_id": "f", + "provider": "anthropic", + "result": {"status": "not_applicable", "reason": "ditto"}, + }, + ] + matrix = build_matrix( + manifest=manifest, + results=results, + litellm_version="v", + claude_code_version="c", + generated_at="t", + ) + assert matrix["features"][0]["providers"]["anthropic"] == { + "status": "not_applicable", + "reason": "feature unsupported on this provider", + } + + +def test_build_matrix_fills_not_tested_for_missing_cells(): + manifest = { + "schema_version": "1", + "providers": ["anthropic", "azure"], + "features": [{"id": "f", "name": "F"}], + } + results = [ + {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, + ] + matrix = build_matrix( + manifest=manifest, + results=results, + litellm_version="v", + claude_code_version="c", + generated_at="t", + ) + cells = matrix["features"][0]["providers"] + assert cells["anthropic"] == {"status": "pass"} + assert cells["azure"] == {"status": "not_tested"} + + +def test_build_matrix_preserves_provider_and_feature_order(): + manifest = { + "schema_version": "1", + "providers": ["azure", "anthropic", "vertex_ai"], + "features": [ + {"id": "z", "name": "Z"}, + {"id": "a", "name": "A"}, + ], + } + matrix = build_matrix( + manifest=manifest, + results=[], + litellm_version="v", + claude_code_version="c", + generated_at="t", + ) + assert matrix["providers"] == ["azure", "anthropic", "vertex_ai"] + assert [f["id"] for f in matrix["features"]] == ["z", "a"] + assert list(matrix["features"][0]["providers"].keys()) == [ + "azure", + "anthropic", + "vertex_ai", + ] + + +def test_build_matrix_emits_schema_version_one(): + manifest = { + "schema_version": "1", + "providers": ["anthropic"], + "features": [{"id": "f", "name": "F"}], + } + matrix = build_matrix( + manifest=manifest, + results=[], + litellm_version="v", + claude_code_version="c", + generated_at="t", + ) + assert matrix["schema_version"] == "1" + + +def test_load_manifest_rejects_wrong_schema_version(tmp_path): + bad = tmp_path / "manifest.yaml" + bad.write_text( + 'schema_version: "2"\nproviders: [anthropic]\nfeatures:\n - id: f\n name: F\n' + ) + with pytest.raises(ManifestError, match="schema_version"): + load_manifest(bad) + + +def test_load_manifest_rejects_empty_features(tmp_path): + bad = tmp_path / "manifest.yaml" + bad.write_text('schema_version: "1"\nproviders: [anthropic]\nfeatures: []\n') + with pytest.raises(ManifestError): + load_manifest(bad) + + +def test_load_results_rejects_missing_results_key(tmp_path): + bad = tmp_path / "results.json" + bad.write_text(json.dumps({"schema_version": "1"})) + with pytest.raises(ResultsError): + load_results(bad) + + +def test_build_matrix_6x5_grid_matches_published_sample(): + """Slice 5 acceptance: feeding the per-model results the full v0 + row set produces reproduces the hand-authored 6x5 sample that the + docs page renders. + + Inputs mirror the structure of `compat-results.json` after a real + run with the proxy configured for all five columns and all six + feature directories: every (feature, provider, model) cell yields a + `pass`. Anthropic announced Claude in Microsoft Foundry on + 2025-11-18, so the Azure column is now exercised end-to-end like + the others rather than reporting `not_applicable`. + + The aggregated matrix must equal the checked-in + `sample_compatibility-matrix.json` byte-for-byte (after JSON load), + so any future schema drift surfaces here in review. + """ + repo_root = Path(__file__).resolve().parents[1] + full_manifest = load_manifest(repo_root / "manifest.yaml") + + # The v0 sample matrix is a frozen baseline: it covers exactly the + # six features the PRD shipped with, in their canonical order. The + # live manifest may carry additional rows (extensions added after + # v0 shipped), but the sample is derived only from the v0 slice so + # this test stays a meaningful regression gate for the v0 cell + # shape rather than chasing every new row added downstream. + v0_feature_ids = [ + "basic_messaging_non_streaming", + "basic_messaging_streaming", + "tool_use", + "prompt_caching_5m", + "vision", + # Row 6 of the v0 PRD; originally shipped as `extended_thinking`. + # The id was renamed in-place to `thinking` to match Anthropic's + # current docs (which reserve "extended thinking" for the + # deprecated manual mode only). The row's *position* in v0 is + # the load-bearing invariant, not the id string. + "thinking", + ] + v0_features = [ + feature + for feature in full_manifest["features"] + if feature["id"] in v0_feature_ids + ] + manifest = {**full_manifest, "features": v0_features} + + feature_ids = [feature["id"] for feature in manifest["features"]] + providers = manifest["providers"] + models = ["claude-haiku-4-5", "claude-sonnet-4-6", "claude-opus-4-7"] + + results = [] + for feature_id in feature_ids: + for provider in providers: + for model in models: + results.append( + { + "feature_id": feature_id, + "provider": provider, + "nodeid": ( + f"tests/e2e/claude_code/{feature_id}/test_{provider}.py" + f"::test[{model}]" + ), + "result": {"status": "pass"}, + } + ) + + matrix = build_matrix( + manifest=manifest, + results=results, + litellm_version="v1.83.0-stable", + claude_code_version="2.1.120", + generated_at="2026-04-25T00:00:00Z", + ) + expected = json.loads((repo_root / "sample_compatibility-matrix.json").read_text()) + assert matrix == expected + + +def test_build_matrix_1x5_grid_one_failing_model_breaks_cell(): + """If even one of three models fails on a provider, that cell is fail + and the error string carries the failing model id so the docs + tooltip can name the outlier.""" + repo_root = Path(__file__).resolve().parents[1] + manifest = load_manifest(repo_root / "manifest.yaml") + + results = [ + { + "feature_id": "basic_messaging_non_streaming", + "provider": "bedrock_invoke", + "result": {"status": "pass"}, + }, + { + "feature_id": "basic_messaging_non_streaming", + "provider": "bedrock_invoke", + "result": { + "status": "fail", + "error": "[claude-opus-4-7-bedrock-invoke] claude CLI exited 1: throttled", + }, + }, + { + "feature_id": "basic_messaging_non_streaming", + "provider": "bedrock_invoke", + "result": {"status": "pass"}, + }, + ] + + matrix = build_matrix( + manifest=manifest, + results=results, + litellm_version="v", + claude_code_version="c", + generated_at="t", + ) + cell = matrix["features"][0]["providers"]["bedrock_invoke"] + assert cell["status"] == "fail" + assert "claude-opus-4-7-bedrock-invoke" in cell["error"] + + +def test_build_from_paths_writes_output(tmp_path): + out = tmp_path / "compatibility-matrix.json" + matrix = build_from_paths( + manifest_path=FIXTURES / "manifest.yaml", + results_path=FIXTURES / "results.json", + litellm_version="v1.83.0-stable", + claude_code_version="2.1.120", + generated_at="2026-04-25T00:00:00Z", + output_path=out, + ) + assert out.exists() + on_disk = json.loads(out.read_text()) + assert on_disk == matrix + expected = json.loads((FIXTURES / "expected_matrix.json").read_text()) + assert on_disk == expected diff --git a/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py b/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py new file mode 100644 index 00000000000..b1745008fac --- /dev/null +++ b/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py @@ -0,0 +1,183 @@ +"""Structural tests for the full v0 6x5 matrix layout. + +These tests don't run the `claude` CLI — they only verify that the +shape of the test suite on disk matches what the PRD declares: six +features in the prescribed order, and for each feature a directory +with one test file per provider column. + +Catching layout drift here means the daily-cron VM and the PR gate +both see the same row set the docs page declares. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +MANIFEST_PATH = REPO_ROOT / "manifest.yaml" + +# The PRD's "Features in v0" section, in row order. +EXPECTED_FEATURE_IDS = [ + "basic_messaging_non_streaming", + "basic_messaging_streaming", + "tool_use", + "prompt_caching_5m", + "vision", + # v0 originally shipped this row as `extended_thinking`. It was + # renamed in-place to `thinking` because Anthropic's docs reserve + # "extended thinking" for the deprecated manual API mode only; the + # single row exercises both manual and adaptive shapes since Claude + # Code picks per model. The PRD's "v0" identity is the *position* + # (row 6, 0-indexed 5), not the id string. + "thinking", +] + +# The PRD's column order. Every feature directory must have one +# `test_.py` for each of these. +EXPECTED_PROVIDERS = [ + "anthropic", + "bedrock_invoke", + "bedrock_converse", + "vertex_ai", + "azure", +] + + +def _all_manifest_feature_ids() -> list[str]: + """Every feature_id currently declared in `manifest.yaml`. + + Evaluated at import time so the result can drive parametrized + structural tests below. Used to catch layout drift on post-v0 + feature rows added after the matrix shipped — the v0 anchor + constants above only validate the original six rows by design. + """ + return [ + feature["id"] + for feature in yaml.safe_load(MANIFEST_PATH.read_text())["features"] + ] + + +ALL_FEATURE_IDS = _all_manifest_feature_ids() + + +@pytest.fixture(scope="module") +def manifest() -> dict: + return yaml.safe_load(MANIFEST_PATH.read_text()) + + +def test_manifest_lists_all_six_v0_features_in_order(manifest): + """The PRD's v0 row set must appear at the top of the manifest in + order. Features beyond v0 (extensions added after the matrix + shipped) are allowed but must not reorder or displace the v0 + rows — the docs page anchors row links by index, so v0 stays + pinned at positions [0:6] for the lifetime of the schema. + """ + ids = [feature["id"] for feature in manifest["features"]] + assert ids[: len(EXPECTED_FEATURE_IDS)] == EXPECTED_FEATURE_IDS + + +def test_manifest_lists_all_five_v0_providers_in_order(manifest): + assert manifest["providers"] == EXPECTED_PROVIDERS + + +def test_manifest_every_feature_has_human_readable_name(manifest): + for feature in manifest["features"]: + assert isinstance(feature["name"], str) and feature["name"].strip() + + +@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) +def test_feature_directory_exists(feature_id): + feature_dir = REPO_ROOT / feature_id + assert feature_dir.is_dir(), f"missing feature directory: {feature_dir}" + + +@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) +@pytest.mark.parametrize("provider", EXPECTED_PROVIDERS) +def test_per_provider_test_file_exists(feature_id, provider): + test_file = REPO_ROOT / feature_id / f"test_{provider}.py" + assert test_file.is_file(), f"missing per-provider test file: {test_file}" + + +@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) +def test_feature_directory_has_init_file(feature_id): + """Each feature directory needs an __init__.py so pytest collects + the per-provider test files as a package — matches the layout + established by `basic_messaging_non_streaming/`.""" + init_file = REPO_ROOT / feature_id / "__init__.py" + assert init_file.is_file(), f"missing __init__.py: {init_file}" + + +# Manifest-driven structural tests: every feature in `manifest.yaml` +# (v0 and post-v0 alike) must have the expected on-disk layout. The +# v0-only tests above pin the position of the original six rows; these +# extend the same structural guarantees to any row added afterward so +# a broken post-v0 directory still fails CI. +@pytest.mark.parametrize("feature_id", ALL_FEATURE_IDS) +def test_every_manifest_feature_has_directory(feature_id): + feature_dir = REPO_ROOT / feature_id + assert feature_dir.is_dir(), ( + f"manifest declares {feature_id!r} but {feature_dir} is missing — " + "feature_id MUST match its on-disk directory (see manifest.yaml header)." + ) + + +@pytest.mark.parametrize("feature_id", ALL_FEATURE_IDS) +def test_every_manifest_feature_has_init_file(feature_id): + init_file = REPO_ROOT / feature_id / "__init__.py" + assert init_file.is_file(), f"missing __init__.py: {init_file}" + + +@pytest.mark.parametrize("feature_id", ALL_FEATURE_IDS) +@pytest.mark.parametrize("provider", EXPECTED_PROVIDERS) +def test_every_manifest_feature_has_per_provider_test_file(feature_id, provider): + """Every (feature, provider) cell in the rendered matrix must be + backed by a per-provider test file. Without this check, a missing + file silently becomes a `not_tested` cell in the published matrix + rather than a CI failure surfacing the layout drift.""" + test_file = REPO_ROOT / feature_id / f"test_{provider}.py" + assert test_file.is_file(), f"missing per-provider test file: {test_file}" + + +@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) +@pytest.mark.parametrize("provider", EXPECTED_PROVIDERS) +def test_per_provider_test_file_imports_and_parametrizes_three_models( + feature_id, provider +): + """Every test file must reference the three Claude tiers required + by the PRD: Haiku 4.5, Sonnet 4.6, Opus 4.7. Implementations may + use plain aliases or per-provider-suffixed aliases (e.g. + `claude-opus-4-7-bedrock-invoke`), so we check for the tier + substrings rather than exact alias names.""" + text = (REPO_ROOT / feature_id / f"test_{provider}.py").read_text() + for tier in ("haiku-4-5", "sonnet-4-6", "opus-4-7"): + assert ( + tier in text + ), f"{feature_id}/test_{provider}.py does not reference {tier}" + + +@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) +def test_azure_test_file_drives_the_proxy(feature_id): + """Azure (Microsoft Foundry) hosts Anthropic Claude as of 2025-11-18, + so every Azure cell in the v0 matrix exercises a real route through + the LiteLLM proxy — same shape as the other provider columns. Pin + that here so a future regression doesn't silently revert these + cells to the old `not_applicable` boilerplate. + + We accept either the direct `run_claude(...)` family of entrypoints + or a per-feature shared helper (e.g. `run_basic_messaging_cell`) + that wraps them — both shapes drive the proxy, and we don't want + this layout pin to block legitimate de-duplication of test bodies. + """ + text = (REPO_ROOT / feature_id / "test_azure.py").read_text() + assert "run_claude" in text or "run_basic_messaging_cell" in text, ( + f"{feature_id}/test_azure.py must drive the claude CLI via run_claude() " + "or a shared helper that wraps it; the not_applicable stub was removed " + "when Foundry started hosting Claude." + ) + assert '"status": "not_applicable"' not in text, ( + f"{feature_id}/test_azure.py still reports not_applicable; Microsoft Foundry " + "now hosts Claude (Haiku 4.5, Sonnet 4.6, Opus 4.7), so this row must run." + ) diff --git a/tests/e2e/claude_code/_driver_unit_tests/__init__.py b/tests/e2e/claude_code/_driver_unit_tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/_driver_unit_tests/conftest.py b/tests/e2e/claude_code/_driver_unit_tests/conftest.py new file mode 100644 index 00000000000..bfeaa57c736 --- /dev/null +++ b/tests/e2e/claude_code/_driver_unit_tests/conftest.py @@ -0,0 +1,32 @@ +"""Local conftest for the driver unit tests. + +Installs a hermetic, no-op rate limiter for every test in this +subdirectory. Without this, importing `cli_driver` and calling +`run_claude(..., runner=fake)` would silently consume tokens from the +shared default limiter (which writes to `$TMPDIR/...`), polluting the +on-disk state another test run might rely on and adding flakiness if +the env vars say "rate=0.1/s". + +A no-op limiter (rate=0 for every provider) returns immediately from +`acquire(...)`, so unit tests behave exactly as they did before the +limiter was added. +""" + +from __future__ import annotations + +import pytest + +from claude_code.rate_limiter import ( + ALL_PROVIDERS, + ProviderConfig, + RateLimiter, + use_limiter, +) + + +@pytest.fixture(autouse=True) +def _hermetic_rate_limiter(tmp_path): + config = {p: ProviderConfig(rate_per_sec=0.0, burst=0.0) for p in ALL_PROVIDERS} + limiter = RateLimiter(config=config, state_dir=tmp_path) + with use_limiter(limiter): + yield diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_basic_messaging.py b/tests/e2e/claude_code/_driver_unit_tests/test_basic_messaging.py new file mode 100644 index 00000000000..018121a8e5c --- /dev/null +++ b/tests/e2e/claude_code/_driver_unit_tests/test_basic_messaging.py @@ -0,0 +1,201 @@ +"""Unit tests for the shared `run_basic_messaging_cell` helper. + +These tests mock `run_claude_models_parallel` so they exercise the +helper's branching (env-missing guard, per-model pass/fail/empty-text, +streaming wire check) without spawning the real CLI. The streaming +check is the regression we care about: a proxy that buffers the +upstream stream must turn the cell red, not green. +""" + +from __future__ import annotations + +import os +from typing import Any, Dict, List, Mapping, Optional, Sequence + +import pytest + +from claude_code import _basic_messaging +from claude_code._basic_messaging import ( + MIN_STREAM_DELTA_EVENTS, + _count_stream_event_deltas, + run_basic_messaging_cell, +) +from claude_code.cli_driver import DriverResult + + +class _FakeResult: + """Stand-in for the test's `compat_result` fixture. + + Records every `set` / `add` payload so assertions can inspect what + the cell reported, in order, without needing the real + `pytest_runtest_logreport` plumbing from `conftest.py`. + """ + + def __init__(self) -> None: + self.rows: List[Dict[str, Any]] = [] + self.single: Optional[Dict[str, Any]] = None + + def set(self, payload: Mapping[str, Any]) -> None: + self.single = dict(payload) + + def add(self, payload: Mapping[str, Any]) -> None: + self.rows.append(dict(payload)) + + +def _streamed_events(n_deltas: int = 5) -> List[Dict[str, Any]]: + """Build a stream-json event list that *looks* streamed. + + Includes `n_deltas` `stream_event` records (matching what + `--include-partial-messages` produces) plus the usual + `system`/`assistant`/`result` boilerplate the CLI always emits. + """ + events: List[Dict[str, Any]] = [{"type": "system", "subtype": "init"}] + for i in range(n_deltas): + events.append( + { + "type": "stream_event", + "event": { + "type": "content_block_delta", + "delta": {"type": "text_delta", "text": str(i)}, + }, + } + ) + events.append( + { + "type": "assistant", + "message": {"content": [{"type": "text", "text": "1\n2\n3"}]}, + } + ) + events.append({"type": "result"}) + return events + + +def _buffered_events() -> List[Dict[str, Any]]: + """Event list a buffering proxy would produce: zero `stream_event`s.""" + return [ + {"type": "system", "subtype": "init"}, + { + "type": "assistant", + "message": {"content": [{"type": "text", "text": "1\n2\n3"}]}, + }, + {"type": "result"}, + ] + + +def _install_fake_runner(monkeypatch, *, outcomes_by_model): + """Patch `run_claude_models_parallel` to return canned outcomes. + + Captures the kwargs the cell passed in so tests can assert on + `extra_args` (which is how the streaming variant opts into + `--include-partial-messages`). + """ + captured: Dict[str, Any] = {} + + def fake(*, models, prompt, base_url, api_key, extra_args=None, **_kwargs): + captured["models"] = list(models) + captured["prompt"] = prompt + captured["base_url"] = base_url + captured["api_key"] = api_key + captured["extra_args"] = list(extra_args) if extra_args else [] + return {model: outcomes_by_model[model] for model in models} + + monkeypatch.setattr(_basic_messaging, "run_claude_models_parallel", fake) + return captured + + +@pytest.fixture(autouse=True) +def _proxy_env(monkeypatch): + monkeypatch.setenv("LITELLM_PROXY_BASE_URL", "http://localhost:4000") + monkeypatch.setenv("LITELLM_PROXY_API_KEY", "sk-test") + + +def test_count_stream_event_deltas_only_counts_records_with_event_payload(): + events = [ + {"type": "system"}, + {"type": "stream_event", "event": {"type": "message_start"}}, + {"type": "stream_event", "event": {"type": "content_block_delta"}}, + {"type": "stream_event"}, + {"type": "stream_event", "event": None}, + {"type": "stream_event", "event": "not-a-dict"}, + {"type": "assistant"}, + {"type": "result"}, + ] + assert _count_stream_event_deltas(events) == 2 + + +def test_verify_streaming_passes_when_proxy_streams(monkeypatch): + fake_result = _FakeResult() + model = "claude-haiku-4-5" + outcome = DriverResult(text="1\n2\n3", events=_streamed_events(n_deltas=5)) + captured = _install_fake_runner(monkeypatch, outcomes_by_model={model: outcome}) + + run_basic_messaging_cell( + compat_result=fake_result, + models=[model], + prompt="Count from 1 to 5, one number per line.", + verify_streaming=True, + ) + + assert captured["extra_args"] == ["--include-partial-messages"] + assert fake_result.rows == [{"status": "pass"}] + + +def test_verify_streaming_fails_when_proxy_buffers(monkeypatch): + fake_result = _FakeResult() + model = "claude-haiku-4-5" + outcome = DriverResult(text="1\n2\n3", events=_buffered_events()) + _install_fake_runner(monkeypatch, outcomes_by_model={model: outcome}) + + with pytest.raises(pytest.fail.Exception): + run_basic_messaging_cell( + compat_result=fake_result, + models=[model], + prompt="Count from 1 to 5, one number per line.", + verify_streaming=True, + ) + + assert len(fake_result.rows) == 1 + row = fake_result.rows[0] + assert row["status"] == "fail" + assert "stream_event" in row["error"] + assert f"< {MIN_STREAM_DELTA_EVENTS}" in row["error"] + + +def test_non_streaming_variant_omits_partial_messages_flag(monkeypatch): + """Default `verify_streaming=False` keeps the non-streaming wire identical.""" + fake_result = _FakeResult() + model = "claude-haiku-4-5" + outcome = DriverResult(text="pong", events=_buffered_events()) + captured = _install_fake_runner(monkeypatch, outcomes_by_model={model: outcome}) + + run_basic_messaging_cell( + compat_result=fake_result, + models=[model], + prompt="Reply with the single word 'pong' and nothing else.", + ) + + assert captured["extra_args"] == [] + assert fake_result.rows == [{"status": "pass"}] + + +def test_verify_streaming_requires_all_models_to_stream(monkeypatch): + """If any one tier buffers, the cell fails — same all-must-pass shape as + the non-streaming check.""" + fake_result = _FakeResult() + outcomes = { + "claude-haiku-4-5": DriverResult(text="ok", events=_streamed_events(5)), + "claude-sonnet-4-6": DriverResult(text="ok", events=_buffered_events()), + "claude-opus-4-7": DriverResult(text="ok", events=_streamed_events(5)), + } + _install_fake_runner(monkeypatch, outcomes_by_model=outcomes) + + with pytest.raises(pytest.fail.Exception): + run_basic_messaging_cell( + compat_result=fake_result, + models=list(outcomes.keys()), + prompt="Count from 1 to 5, one number per line.", + verify_streaming=True, + ) + + statuses = [row["status"] for row in fake_result.rows] + assert statuses == ["pass", "fail", "pass"] diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py b/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py new file mode 100644 index 00000000000..786f6029993 --- /dev/null +++ b/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py @@ -0,0 +1,795 @@ +"""Unit tests for the Claude Code CLI Driver. + +These tests mock the subprocess so they run anywhere — no network, no +`claude` install, no API keys. They cover the behavior contract: +argument assembly, environment overlay, stream-JSON parsing, exit-code +plumbing, and the structured failure modes (CLI not found, timeout). +""" + +from __future__ import annotations + +import json +import subprocess +from dataclasses import dataclass +from typing import List, Optional + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + DriverResult, + failure_diagnostic, + run_claude, + run_claude_models_parallel, +) + + +@dataclass +class _Completed: + returncode: int = 0 + stdout: str = "" + stderr: str = "" + + +def _make_runner(*, stdout: str = "", returncode: int = 0, stderr: str = ""): + captured = {} + + def runner(cmd, env, capture_output, text, timeout, check, input=None): + captured["cmd"] = cmd + captured["env"] = env + captured["timeout"] = timeout + captured["input"] = input + return _Completed(returncode=returncode, stdout=stdout, stderr=stderr) + + return runner, captured + + +def test_run_claude_assembles_command_correctly(): + runner, captured = _make_runner( + stdout='{"type":"assistant","message":{"content":[{"type":"text","text":"ok"}]}}\n' + ) + run_claude( + prompt="hello", + model="claude-haiku-4-5", + base_url="http://localhost:4000", + api_key="sk-test", + runner=runner, + ) + cmd = captured["cmd"] + assert cmd[0] == "claude" + assert "--print" in cmd + assert "--output-format" in cmd + assert "stream-json" in cmd + assert "--model" in cmd + assert "claude-haiku-4-5" in cmd + # prompt is the last positional after the `--` end-of-options marker. + assert cmd[-2:] == ["--", "hello"] + + +def test_run_claude_places_extra_args_before_prompt(): + """`claude --print` expects the prompt as the final positional arg. + + Flags appearing after the prompt are ignored or eaten by the prompt + parser (especially variadic flags like `--allowed-tools `), + which silently broke the tool_use, vision, and web_search cells + before the fix. Pin the ordering: every flag (including + caller-supplied `extra_args`) must precede the `--` end-of-options + marker, which itself precedes the prompt. + """ + runner, captured = _make_runner(stdout="") + run_claude( + prompt="say hi", + model="claude-haiku-4-5", + base_url="http://localhost:4000", + api_key="sk-test", + extra_args=["--allowed-tools", "Bash"], + runner=runner, + ) + cmd = captured["cmd"] + # Prompt is last, `--` immediately precedes it, and the caller's + # extra_args sit somewhere earlier in the command. + assert cmd[-2:] == ["--", "say hi"] + assert "--allowed-tools" in cmd + assert cmd.index("--allowed-tools") < cmd.index("--") + + +def test_run_claude_overlays_proxy_env(): + runner, captured = _make_runner(stdout="") + run_claude( + prompt="hi", + model="claude-opus-4-7", + base_url="http://proxy.example:4000", + api_key="sk-abc", + runner=runner, + ) + env = captured["env"] + assert env["ANTHROPIC_BASE_URL"] == "http://proxy.example:4000" + assert env["ANTHROPIC_AUTH_TOKEN"] == "sk-abc" + + +def test_run_claude_extra_env_is_added_to_subprocess_env(): + """Caller-supplied extra_env entries land on the subprocess env.""" + runner, captured = _make_runner(stdout="") + run_claude( + prompt="hi", + model="claude-opus-4-7", + base_url="http://localhost", + api_key="sk-abc", + extra_env={"MAX_THINKING_TOKENS": "4096"}, + runner=runner, + ) + assert captured["env"]["MAX_THINKING_TOKENS"] == "4096" + + +def test_run_claude_inherits_only_allowlisted_os_environ(monkeypatch): + """Process-runtime vars (PATH) flow through; credentials don't. + + The `claude` CLI is a Node binary installed dynamically from npm in + CI. If the package were ever compromised, inheriting the entire + parent environment would hand it every credential the surrounding + proxy job loads (AWS keys, Azure Foundry key, GitHub token, etc.). + Pin the contract: only the small allowlist of runtime vars is + inherited; everything else is dropped unless the caller passes it + explicitly via extra_env. + + `HOME` is *not* on the allowlist anymore — see the dedicated + isolated-HOME test below for the reason. + """ + monkeypatch.setenv("PATH", "/usr/bin:/usr/local/bin") + monkeypatch.setenv("HOME", "/home/runner") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "totally-secret") + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-proxy-only") + monkeypatch.setenv("AZURE_FOUNDRY_API_KEY", "azure-secret") + monkeypatch.setenv("VERTEXAI_CREDENTIALS", '{"private_key": "leak"}') + monkeypatch.setenv("GITHUB_TOKEN", "ghs_xxx") + + runner, captured = _make_runner(stdout="") + run_claude( + prompt="hi", + model="claude-opus-4-7", + base_url="http://localhost", + api_key="sk-abc", + runner=runner, + ) + env = captured["env"] + assert env["PATH"] == "/usr/bin:/usr/local/bin" + assert "AWS_SECRET_ACCESS_KEY" not in env + assert "ANTHROPIC_API_KEY" not in env + assert "AZURE_FOUNDRY_API_KEY" not in env + assert "VERTEXAI_CREDENTIALS" not in env + assert "GITHUB_TOKEN" not in env + + +def test_run_claude_uses_isolated_per_invocation_home(monkeypatch, tmp_path): + """`claude` subprocess never sees the runtime user's real $HOME. + + The CLI needs *a* HOME (it caches per-session state under + `$HOME/.claude/projects//`), but it has no business reading + the runtime user's real one. On the cron VM the runtime user is a + real interactive account with a populated home directory + (~/.config/gh/hosts.yml carrying a GitHub token, ~/.ssh/, etc.); + handing /home/mateo to a compromised npm package — or to a + model-directed `Read` tool call during the PDF/vision cells — + would let it exfiltrate those files. We hand the CLI a fresh + empty per-invocation tmpdir instead. + """ + monkeypatch.setenv("HOME", "/home/runner") + + runner, captured = _make_runner(stdout="") + run_claude( + prompt="hi", + model="claude-opus-4-7", + base_url="http://localhost", + api_key="sk-abc", + runner=runner, + ) + env = captured["env"] + assert "HOME" in env, "claude CLI needs HOME to find ~/.claude session dir" + assert ( + env["HOME"] != "/home/runner" + ), "HOME must not leak the parent process's HOME to claude" + # The isolated HOME is a fresh tmpdir prefixed `claude-cli-home-`; + # see `_make_isolated_home` in cli_driver.py. It exists during the + # subprocess call and is removed afterwards (cleanup runs in a + # `finally`, so by the time this assertion runs the dir is gone — + # we only check the *prefix* of the path string we captured). + assert "claude-cli-home-" in env["HOME"] + + +def test_run_claude_isolated_home_is_distinct_per_invocation(monkeypatch): + """Two consecutive calls get two different isolated HOMEs. + + Reusing a single tmpdir across calls would defeat the isolation + in the parallel matrix run (a compromised CLI could plant a file + in HOME on one model's run and read it on the next). Pin: each + `run_claude` invocation gets its own freshly-created HOME. + """ + monkeypatch.setenv("HOME", "/home/runner") + + runner, captured = _make_runner(stdout="") + run_claude( + prompt="hi", + model="claude-opus-4-7", + base_url="http://localhost", + api_key="sk-abc", + runner=runner, + ) + home_a = captured["env"]["HOME"] + + runner, captured = _make_runner(stdout="") + run_claude( + prompt="hi", + model="claude-opus-4-7", + base_url="http://localhost", + api_key="sk-abc", + runner=runner, + ) + home_b = captured["env"]["HOME"] + + assert home_a != home_b + + +def test_run_claude_isolated_home_cleaned_up_after_run(monkeypatch): + """The per-invocation HOME tmpdir is rm-rf'd when run_claude returns. + + Without cleanup, a long matrix run would accumulate one tmpdir + per cell × per model × per CLI call (~75 dirs per cron run, + growing without bound across days). + """ + import os as _os + + monkeypatch.setenv("HOME", "/home/runner") + + runner, captured = _make_runner(stdout="") + run_claude( + prompt="hi", + model="claude-opus-4-7", + base_url="http://localhost", + api_key="sk-abc", + runner=runner, + ) + isolated_home = captured["env"]["HOME"] + assert not _os.path.exists( + isolated_home + ), f"isolated HOME {isolated_home!r} should be removed after run_claude returns" + + +def test_run_claude_isolated_home_cleaned_up_on_subprocess_failure(monkeypatch): + """Cleanup runs even when the CLI subprocess raises. + + If the CLI is missing or times out, `run_claude` raises + `ClaudeCLIError` — but the per-invocation HOME tmpdir must still + be removed (the `finally` clause), otherwise long failure-prone + runs leak tmpdirs. + """ + import os as _os + + monkeypatch.setenv("HOME", "/home/runner") + + captured: dict = {} + + def runner(cmd, env, capture_output, text, timeout, check, input=None): + captured["env"] = env + raise subprocess.TimeoutExpired(cmd=cmd, timeout=timeout) + + with pytest.raises(ClaudeCLIError): + run_claude( + prompt="hi", + model="claude-opus-4-7", + base_url="http://localhost", + api_key="sk-abc", + runner=runner, + ) + + isolated_home = captured["env"]["HOME"] + assert not _os.path.exists( + isolated_home + ), f"isolated HOME {isolated_home!r} should be removed even on timeout" + + +def test_run_claude_extra_env_can_pass_through_otherwise_blocked_var(monkeypatch): + """The allowlist applies to inherited os.environ; extra_env is the + sanctioned way for a test to opt-in to passing something extra.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "from-os") + runner, captured = _make_runner(stdout="") + run_claude( + prompt="hi", + model="claude-opus-4-7", + base_url="http://localhost", + api_key="sk-abc", + extra_env={"ANTHROPIC_API_KEY": "from-arg"}, + runner=runner, + ) + assert captured["env"]["ANTHROPIC_API_KEY"] == "from-arg" + + +def test_run_claude_parses_stream_json_assistant_text(): + events = [ + {"type": "system", "session_id": "abc"}, + { + "type": "assistant", + "message": { + "content": [ + {"type": "text", "text": "Hello "}, + {"type": "text", "text": "world"}, + ] + }, + }, + {"type": "result", "usage": {"input_tokens": 10, "output_tokens": 2}}, + ] + stdout = "\n".join(json.dumps(e) for e in events) + "\n" + runner, _ = _make_runner(stdout=stdout) + result = run_claude( + prompt="hi", + model="claude-haiku-4-5", + base_url="http://localhost", + api_key="sk-abc", + runner=runner, + ) + assert isinstance(result, DriverResult) + assert result.text == "Hello world" + assert len(result.events) == 3 + assert result.usage == {"input_tokens": 10, "output_tokens": 2} + assert result.exit_code == 0 + + +def test_run_claude_handles_string_message_content(): + """Some CLI versions emit `message.content` as a plain string.""" + stdout = ( + json.dumps({"type": "assistant", "message": {"content": "bare text"}}) + "\n" + ) + runner, _ = _make_runner(stdout=stdout) + result = run_claude( + prompt="hi", + model="m", + base_url="http://x", + api_key="k", + runner=runner, + ) + assert result.text == "bare text" + + +def test_run_claude_skips_malformed_lines(): + stdout = ( + "not-json\n" + + json.dumps( + { + "type": "assistant", + "message": {"content": [{"type": "text", "text": "x"}]}, + } + ) + + "\n" + + "{also-bad\n" + ) + runner, _ = _make_runner(stdout=stdout) + result = run_claude( + prompt="hi", + model="m", + base_url="http://x", + api_key="k", + runner=runner, + ) + assert result.text == "x" + assert len(result.events) == 1 + + +def test_run_claude_propagates_nonzero_exit_code(): + runner, _ = _make_runner(stdout="", returncode=2, stderr="auth failed") + result = run_claude( + prompt="hi", + model="m", + base_url="http://x", + api_key="k", + runner=runner, + ) + assert result.exit_code == 2 + assert result.stderr == "auth failed" + assert result.text == "" + + +def test_run_claude_raises_on_missing_cli(): + def runner(*args, **kwargs): + raise FileNotFoundError(2, "no such file", "claude") + + with pytest.raises(ClaudeCLIError, match="claude CLI not found"): + run_claude( + prompt="hi", + model="m", + base_url="http://x", + api_key="k", + runner=runner, + ) + + +def test_run_claude_raises_on_timeout(): + def runner(*args, **kwargs): + raise subprocess.TimeoutExpired(cmd="claude", timeout=1) + + with pytest.raises(ClaudeCLIError, match="timed out"): + run_claude( + prompt="hi", + model="m", + base_url="http://x", + api_key="k", + timeout=1, + runner=runner, + ) + + +def test_run_claude_validates_required_params(): + runner, _ = _make_runner() + with pytest.raises(ValueError, match="prompt"): + run_claude( + prompt="", + model="m", + base_url="http://x", + api_key="k", + runner=runner, + ) + with pytest.raises(ValueError, match="stdin_input"): + run_claude( + prompt=None, + stdin_input="", + model="m", + base_url="http://x", + api_key="k", + runner=runner, + ) + with pytest.raises(ValueError, match="model"): + run_claude( + prompt="hi", + model="", + base_url="http://x", + api_key="k", + runner=runner, + ) + with pytest.raises(ValueError, match="base_url"): + run_claude( + prompt="hi", + model="m", + base_url="", + api_key="k", + runner=runner, + ) + with pytest.raises(ValueError, match="api_key"): + run_claude( + prompt="hi", + model="m", + base_url="http://x", + api_key="", + runner=runner, + ) + + +# --------------------------------------------------------------------------- +# failure_diagnostic +# +# Regression coverage for the bring-up incident where the proxy was started +# with the wrong config and tests reported only `claude CLI exited 1` while +# the actual 400 from LiteLLM was sitting in stdout. The helper must surface +# api_status, the assistant text (where API errors land), stderr, and the +# exit code together — and gracefully degrade when individual pieces are +# missing. +# --------------------------------------------------------------------------- + + +def test_failure_diagnostic_surfaces_api_error_text_from_stdout(): + """The CLI hides 4xx/5xx from the proxy in `assistant.message.content` text.""" + api_error_text = ( + 'API Error: 400 {"error":{"message":"litellm.BadRequestError: ' + "You passed in model=claude-haiku-4-5. There are no healthy " + 'deployments..."}}' + ) + result = DriverResult( + text=api_error_text, + events=[ + { + "type": "assistant", + "message": {"content": [{"type": "text", "text": api_error_text}]}, + }, + { + "type": "result", + "is_error": True, + "api_error_status": 400, + "result": api_error_text, + }, + ], + exit_code=1, + stderr="", + ) + + diag = failure_diagnostic(result) + + assert "exit=1" in diag + assert "api_status=400" in diag + assert "There are no healthy deployments" in diag + + +def test_failure_diagnostic_falls_back_to_stderr_when_no_text(): + result = DriverResult(text="", events=[], exit_code=2, stderr="boom\n") + diag = failure_diagnostic(result) + assert "exit=2" in diag + assert "stderr=boom" in diag + + +def test_failure_diagnostic_handles_completely_empty_result(): + """A run that produced literally nothing should still yield a useful string.""" + result = DriverResult(text="", events=[], exit_code=137, stderr="") + diag = failure_diagnostic(result) + assert "exit=137" in diag + assert "no diagnostic output" in diag + + +def test_failure_diagnostic_truncates_long_text(): + """Don't let a 5MB HTML 502 page from a load balancer wreck the matrix JSON.""" + huge = "x" * 5000 + result = DriverResult(text=huge, events=[], exit_code=1, stderr="") + diag = failure_diagnostic(result, max_len=100) + assert "truncated" in diag + # Allow some slack for the prefix/suffix/separator characters. + assert len(diag) < 300 + + +def test_failure_diagnostic_ignores_non_int_api_error_status(): + """The CLI sometimes emits api_error_status as a string; don't crash.""" + result = DriverResult( + text="oops", + events=[{"type": "result", "api_error_status": "n/a"}], + exit_code=1, + stderr="", + ) + diag = failure_diagnostic(result) + assert "api_status" not in diag + assert "text=oops" in diag + + +# --------------------------------------------------------------------------- +# run_claude_models_parallel +# +# The matrix runs three Claude tiers per cell, so the parallel helper has to +# (a) invoke `run_claude` once per model, (b) preserve each model's outcome +# separately, and (c) return errors as values rather than raising — callers +# need both the failed and the succeeded model results to report per-cell +# rows accurately. +# --------------------------------------------------------------------------- + + +def test_run_claude_models_parallel_returns_one_result_per_model(): + """Each model gets its own DriverResult keyed under the helper's dict.""" + seen_models: List[str] = [] + + def runner(cmd, env, capture_output, text, timeout, check, input=None): + # The model id is two slots after `--model` in the assembled command. + idx = cmd.index("--model") + model = cmd[idx + 1] + seen_models.append(model) + return _Completed( + returncode=0, + stdout=json.dumps( + { + "type": "assistant", + "message": { + "content": [{"type": "text", "text": f"reply-{model}"}] + }, + } + ) + + "\n", + ) + + outcomes = run_claude_models_parallel( + models=["a", "b", "c"], + prompt="hi", + base_url="http://x", + api_key="k", + runner=runner, + ) + + assert set(outcomes.keys()) == {"a", "b", "c"} + for model in ("a", "b", "c"): + result = outcomes[model] + assert isinstance(result, DriverResult) + assert result.text == f"reply-{model}" + assert result.exit_code == 0 + assert sorted(seen_models) == ["a", "b", "c"] + + +def test_run_claude_models_parallel_returns_errors_as_values(): + """A model whose CLI is missing surfaces as a ClaudeCLIError, not a raise.""" + + def runner(cmd, env, capture_output, text, timeout, check, input=None): + idx = cmd.index("--model") + model = cmd[idx + 1] + if model == "boom": + raise FileNotFoundError(2, "no such file", "claude") + return _Completed( + returncode=0, + stdout=json.dumps( + { + "type": "assistant", + "message": {"content": [{"type": "text", "text": "ok"}]}, + } + ) + + "\n", + ) + + outcomes = run_claude_models_parallel( + models=["ok-model", "boom"], + prompt="hi", + base_url="http://x", + api_key="k", + runner=runner, + ) + + assert isinstance(outcomes["ok-model"], DriverResult) + assert outcomes["ok-model"].text == "ok" + assert isinstance(outcomes["boom"], ClaudeCLIError) + assert "claude CLI not found" in str(outcomes["boom"]) + + +def test_run_claude_models_parallel_preserves_nonzero_exit_codes(): + """Mixed success/failure on exit code should not collapse into one verdict.""" + + def runner(cmd, env, capture_output, text, timeout, check, input=None): + idx = cmd.index("--model") + model = cmd[idx + 1] + if model == "fail": + return _Completed(returncode=2, stdout="", stderr="auth failed") + return _Completed( + returncode=0, + stdout=json.dumps( + { + "type": "assistant", + "message": {"content": [{"type": "text", "text": "ok"}]}, + } + ) + + "\n", + ) + + outcomes = run_claude_models_parallel( + models=["ok-model", "fail"], + prompt="hi", + base_url="http://x", + api_key="k", + runner=runner, + ) + + assert outcomes["ok-model"].exit_code == 0 + assert outcomes["fail"].exit_code == 2 + assert outcomes["fail"].stderr == "auth failed" + + +def test_run_claude_models_parallel_rejects_empty_models(): + with pytest.raises(ValueError, match="non-empty"): + run_claude_models_parallel( + models=[], + prompt="hi", + base_url="http://x", + api_key="k", + ) + + +def test_run_claude_models_parallel_stamps_duration_on_each_result(): + """Each DriverResult carries the per-model wall time so callers can + attribute slow cells without re-timing the work themselves. + + The fake runner sleeps for very different durations per model so + we can prove each result is timing its own work (not the batch + wall time). We use generous absolute bounds because thread-pool + scheduling on a loaded CI box adds noise on the order of tens of + milliseconds. + """ + import time + + def runner(cmd, env, capture_output, text, timeout, check, input=None): + idx = cmd.index("--model") + model = cmd[idx + 1] + time.sleep(0.05 if model == "fast" else 0.40) + return _Completed(returncode=0, stdout="") + + outcomes = run_claude_models_parallel( + models=["fast", "slow"], + prompt="hi", + base_url="http://x", + api_key="k", + runner=runner, + ) + + fast_ms = outcomes["fast"].duration_ms + slow_ms = outcomes["slow"].duration_ms + assert fast_ms is not None and slow_ms is not None + # 50ms sleep ⇒ ~50–250ms after scheduling overhead; 400ms sleep ⇒ + # 400–700ms. We just need the two distributions to be non-overlapping + # so we know each row's duration is its own work, not the batch's. + assert fast_ms < 300, fast_ms + assert slow_ms >= 350, slow_ms + assert slow_ms > fast_ms + + +def test_run_claude_models_parallel_breakdown_logs_to_stderr(capsys): + """The breakdown helper must emit a per-model timing block so users + can answer "why didn't parallel help?" without re-instrumenting.""" + + def runner(cmd, env, capture_output, text, timeout, check, input=None): + return _Completed(returncode=0, stdout="") + + run_claude_models_parallel( + models=["model-x", "model-y"], + prompt="hi", + base_url="http://x", + api_key="k", + runner=runner, + ) + + captured = capsys.readouterr() + assert "[parallel] per-model wall time:" in captured.err + assert "model-x" in captured.err + assert "model-y" in captured.err + assert "speedup=" in captured.err + assert "slowest=" in captured.err + + +def test_run_claude_models_parallel_breakdown_marks_cli_errors(capsys): + """When a model raises ClaudeCLIError, the breakdown should still + show its row tagged as `cli-error` rather than crashing or omitting it.""" + + def runner(cmd, env, capture_output, text, timeout, check, input=None): + idx = cmd.index("--model") + if cmd[idx + 1] == "boom": + raise FileNotFoundError(2, "no such file", "claude") + return _Completed(returncode=0, stdout="") + + run_claude_models_parallel( + models=["ok-model", "boom"], + prompt="hi", + base_url="http://x", + api_key="k", + runner=runner, + ) + + captured = capsys.readouterr() + assert "ok-model" in captured.err + assert "boom" in captured.err + assert "cli-error" in captured.err + + +def test_run_claude_models_parallel_forwards_extra_args_and_env(): + """Shared kwargs must reach every per-model invocation unchanged.""" + captured_envs: List[dict] = [] + captured_cmds: List[List[str]] = [] + + def runner(cmd, env, capture_output, text, timeout, check, input=None): + captured_envs.append(env) + captured_cmds.append(cmd) + return _Completed(returncode=0, stdout="") + + run_claude_models_parallel( + models=["a", "b"], + prompt="hi", + base_url="http://x", + api_key="k", + extra_env={"MAX_THINKING_TOKENS": "4096"}, + extra_args=["--allowed-tools", "Bash"], + runner=runner, + ) + + assert all(env["MAX_THINKING_TOKENS"] == "4096" for env in captured_envs) + for cmd in captured_cmds: + assert "--allowed-tools" in cmd + assert "Bash" in cmd + + +def test_failure_diagnostic_uses_last_result_event_status(): + """If multiple `result` events appear, the most recent status wins.""" + result = DriverResult( + text="", + events=[ + {"type": "result", "api_error_status": 500}, + {"type": "assistant", "message": {"content": []}}, + {"type": "result", "api_error_status": 429}, + ], + exit_code=1, + stderr="", + ) + diag = failure_diagnostic(result) + assert "api_status=429" in diag + assert "500" not in diag diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_compat_result.py b/tests/e2e/claude_code/_driver_unit_tests/test_compat_result.py new file mode 100644 index 00000000000..b3a904946d3 --- /dev/null +++ b/tests/e2e/claude_code/_driver_unit_tests/test_compat_result.py @@ -0,0 +1,138 @@ +"""Tests for the `compat_result` fixture's tagged-union validation. + +The conftest's `pytest_runtest_makereport` hook is exercised end-to-end by +the matrix-builder golden-file tests (which consume a results.json that +the harness would produce). Here we just test the input-validation +contract on `CompatResult.set()`. +""" + +from __future__ import annotations + +import pytest + +from claude_code.conftest import CompatResult + + +def test_set_pass_is_accepted(): + r = CompatResult() + r.set({"status": "pass"}) + assert r.value == {"status": "pass"} + + +def test_set_fail_requires_error(): + r = CompatResult() + with pytest.raises(ValueError, match="requires 'error'"): + r.set({"status": "fail"}) + + +def test_set_fail_with_error_is_accepted(): + r = CompatResult() + r.set({"status": "fail", "error": "boom"}) + assert r.value == {"status": "fail", "error": "boom"} + + +def test_set_not_applicable_requires_reason(): + r = CompatResult() + with pytest.raises(ValueError, match="requires 'reason'"): + r.set({"status": "not_applicable"}) + + +def test_set_not_applicable_with_reason_is_accepted(): + r = CompatResult() + r.set({"status": "not_applicable", "reason": "Bedrock has no /thinking"}) + assert r.value == {"status": "not_applicable", "reason": "Bedrock has no /thinking"} + + +def test_set_not_tested_is_accepted(): + r = CompatResult() + r.set({"status": "not_tested"}) + assert r.value == {"status": "not_tested"} + + +def test_set_rejects_unknown_status(): + r = CompatResult() + with pytest.raises(ValueError, match="status must be one of"): + r.set({"status": "maybe"}) + + +def test_set_rejects_non_dict(): + r = CompatResult() + with pytest.raises(TypeError): + r.set("pass") # type: ignore[arg-type] + + +def test_set_copies_input(): + """Mutating the dict after set() must not change the stored value.""" + r = CompatResult() + payload = {"status": "fail", "error": "x"} + r.set(payload) + payload["error"] = "mutated" + assert r.value["error"] == "x" + + +# --------------------------------------------------------------------------- +# add() / collected() +# +# When a single test exercises three Claude tiers in parallel, each tier +# needs its own row in the results artifact so the matrix builder can +# apply its "all three must pass" aggregation. `add()` is the per-tier +# recorder; `collected()` is what the conftest hook reads. +# --------------------------------------------------------------------------- + + +def test_add_appends_each_call_to_values(): + r = CompatResult() + r.add({"status": "pass"}) + r.add({"status": "fail", "error": "bad"}) + assert r.values == [ + {"status": "pass"}, + {"status": "fail", "error": "bad"}, + ] + + +def test_add_validates_like_set(): + """The add() and set() validators are the same; both must reject bad payloads.""" + r = CompatResult() + with pytest.raises(ValueError, match="requires 'error'"): + r.add({"status": "fail"}) + with pytest.raises(ValueError, match="requires 'reason'"): + r.add({"status": "not_applicable"}) + with pytest.raises(ValueError, match="status must be one of"): + r.add({"status": "maybe"}) + with pytest.raises(TypeError): + r.add("pass") # type: ignore[arg-type] + + +def test_add_copies_input(): + """Same defensive copy contract as set().""" + r = CompatResult() + payload = {"status": "fail", "error": "x"} + r.add(payload) + payload["error"] = "mutated" + assert r.values[0]["error"] == "x" + + +def test_collected_returns_values_when_added(): + r = CompatResult() + r.add({"status": "pass"}) + r.add({"status": "pass"}) + assert r.collected() == [{"status": "pass"}, {"status": "pass"}] + + +def test_collected_returns_single_value_when_only_set_called(): + """Legacy single-result tests should still surface their one outcome.""" + r = CompatResult() + r.set({"status": "pass"}) + assert r.collected() == [{"status": "pass"}] + + +def test_collected_prefers_added_values_over_set_value(): + """If both are populated, the per-tier list wins — that's the multi-model shape.""" + r = CompatResult() + r.set({"status": "pass"}) + r.add({"status": "fail", "error": "tier-2 broke"}) + assert r.collected() == [{"status": "fail", "error": "tier-2 broke"}] + + +def test_collected_returns_empty_when_nothing_reported(): + assert CompatResult().collected() == [] diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py b/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py new file mode 100644 index 00000000000..92907eda3c4 --- /dev/null +++ b/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py @@ -0,0 +1,329 @@ +"""Unit tests for the cross-process token-bucket rate limiter. + +The tests cover three layers: + +1. Provider inference from model alias — the matrix-column mapping the + live tests rely on (`-bedrock-converse` vs `-bedrock-invoke` vs + `-azure` vs `-vertex` vs bare = anthropic). + +2. Config parsing — env-var precedence, fallback to default, malformed + input handling, burst override semantics. These run against + `os.environ`-shaped dicts so we don't have to monkeypatch globals. + +3. Token-bucket behavior — enforcing rate, accumulating burst, never + over-spending across a fake clock. Filesystem state is exercised + with a real `tmp_path` because the persistence is the whole point; + the only injected seam is `clock` (and `sleep`, so tests don't + actually wait on wall time). + +The cross-process flock semantics are exercised indirectly: every +test creates a fresh `RateLimiter` rooted at `tmp_path`, so the same +file lock that protects production is exercised here too. We don't +fork to test multi-process behavior in this file because pytest +fixtures + xdist already do that for the integration suite. +""" + +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import List + +import pytest + +from claude_code.rate_limiter import ( + ALL_PROVIDERS, + BURST_ENV, + DEFAULT_RATE, + PROVIDER_ANTHROPIC, + PROVIDER_AZURE, + PROVIDER_BEDROCK_CONVERSE, + PROVIDER_BEDROCK_INVOKE, + PROVIDER_VERTEX_AI, + ProviderConfig, + RateLimiter, + get_default_limiter, + infer_provider, + load_config, + reset_default_limiter, + use_limiter, +) + + +# --------------------------------------------------------------------------- +# Provider inference +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "model, expected", + [ + ("claude-haiku-4-5", PROVIDER_ANTHROPIC), + ("claude-sonnet-4-6", PROVIDER_ANTHROPIC), + ("claude-opus-4-7", PROVIDER_ANTHROPIC), + ("claude-haiku-4-5-azure", PROVIDER_AZURE), + ("claude-sonnet-4-6-azure", PROVIDER_AZURE), + ("claude-opus-4-7-vertex", PROVIDER_VERTEX_AI), + ("claude-haiku-4-5-bedrock-converse", PROVIDER_BEDROCK_CONVERSE), + ("claude-haiku-4-5-bedrock-invoke", PROVIDER_BEDROCK_INVOKE), + ], +) +def test_infer_provider_maps_alias_suffix_to_column(model, expected): + assert infer_provider(model) == expected + + +def test_infer_provider_bedrock_converse_beats_bedrock_invoke_lookup_order(): + """Both bedrock suffixes contain `bedrock`; the more-specific suffix wins.""" + assert infer_provider("claude-foo-bedrock-converse") == PROVIDER_BEDROCK_CONVERSE + assert infer_provider("claude-foo-bedrock-invoke") == PROVIDER_BEDROCK_INVOKE + + +def test_infer_provider_rejects_empty_string(): + with pytest.raises(ValueError, match="non-empty"): + infer_provider("") + + +def test_infer_provider_is_case_insensitive(): + """Aliases in the proxy config sometimes drift between cases; we + should still route them to the right column.""" + assert infer_provider("CLAUDE-OPUS-4-7-AZURE") == PROVIDER_AZURE + + +# --------------------------------------------------------------------------- +# Config parsing +# --------------------------------------------------------------------------- + + +def test_load_config_uses_default_rate_when_env_absent(): + cfg = load_config(env={}) + for provider in ALL_PROVIDERS: + assert cfg[provider].rate_per_sec == DEFAULT_RATE + assert cfg[provider].burst == DEFAULT_RATE + + +def test_load_config_reads_per_provider_rate(): + cfg = load_config( + env={ + "LITELLM_COMPAT_RATE_ANTHROPIC": "10", + "LITELLM_COMPAT_RATE_AZURE": "0.5", + } + ) + assert cfg[PROVIDER_ANTHROPIC].rate_per_sec == 10.0 + assert cfg[PROVIDER_AZURE].rate_per_sec == 0.5 + assert cfg[PROVIDER_VERTEX_AI].rate_per_sec == DEFAULT_RATE + + +def test_load_config_zero_rate_disables_provider(): + cfg = load_config(env={"LITELLM_COMPAT_RATE_BEDROCK_INVOKE": "0"}) + assert cfg[PROVIDER_BEDROCK_INVOKE].enabled is False + + +def test_load_config_burst_override_applies_to_every_provider(): + cfg = load_config( + env={ + "LITELLM_COMPAT_RATE_ANTHROPIC": "5", + BURST_ENV: "20", + } + ) + for provider in ALL_PROVIDERS: + assert cfg[provider].burst == 20.0 + + +def test_load_config_falls_back_on_malformed_value(): + cfg = load_config(env={"LITELLM_COMPAT_RATE_ANTHROPIC": "not-a-number"}) + assert cfg[PROVIDER_ANTHROPIC].rate_per_sec == DEFAULT_RATE + + +def test_load_config_burst_floors_at_one_when_rate_is_low(): + """A 0.5/s rate with no burst override must still allow at least + one immediate request — otherwise the very first call would block.""" + cfg = load_config(env={"LITELLM_COMPAT_RATE_ANTHROPIC": "0.5"}) + assert cfg[PROVIDER_ANTHROPIC].burst == 1.0 + + +# --------------------------------------------------------------------------- +# Token bucket +# --------------------------------------------------------------------------- + + +@pytest.fixture +def fake_clock(): + """A controllable monotonic clock + sleep for the limiter under test. + + Tests advance `clock.now` to simulate elapsed wall time. `sleep` + adds the requested duration to `clock.now` instead of actually + sleeping, so a "wait 200ms" code path runs in microseconds and + is deterministic. + """ + + class Clock: + def __init__(self): + self.now = 1_000.0 + self.sleeps: List[float] = [] + + def __call__(self): + return self.now + + def sleep(self, seconds: float) -> None: + self.sleeps.append(seconds) + self.now += seconds + + return Clock() + + +def _make_limiter(tmp_path: Path, fake_clock, *, rate=10.0, burst=None): + cfg = { + p: ProviderConfig(rate_per_sec=rate, burst=burst if burst is not None else rate) + for p in ALL_PROVIDERS + } + return RateLimiter( + config=cfg, + state_dir=tmp_path, + clock=fake_clock, + sleep=fake_clock.sleep, + ) + + +def test_acquire_first_call_does_not_wait(tmp_path, fake_clock): + """A freshly-initialized bucket starts full; the first acquire is free.""" + limiter = _make_limiter(tmp_path, fake_clock, rate=10.0, burst=10.0) + waited = limiter.acquire(PROVIDER_ANTHROPIC) + assert waited == 0.0 + assert fake_clock.sleeps == [] + + +def test_acquire_disabled_provider_returns_immediately(tmp_path, fake_clock): + """rate=0 ⇒ no throttling, even if every other provider is throttled.""" + cfg = {p: ProviderConfig(rate_per_sec=0.0, burst=0.0) for p in ALL_PROVIDERS} + limiter = RateLimiter( + config=cfg, state_dir=tmp_path, clock=fake_clock, sleep=fake_clock.sleep + ) + for _ in range(100): + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 + assert fake_clock.sleeps == [] + + +def test_acquire_burns_through_burst_then_throttles(tmp_path, fake_clock): + """`burst` immediate requests succeed; the next one waits 1/rate seconds.""" + limiter = _make_limiter(tmp_path, fake_clock, rate=2.0, burst=3.0) + + for _ in range(3): + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 + + # Bucket is empty; next call must sleep ~0.5s to earn one token at 2/s. + waited = limiter.acquire(PROVIDER_ANTHROPIC) + assert waited == pytest.approx(0.5, abs=0.01) + + +def test_acquire_refills_with_elapsed_time(tmp_path, fake_clock): + """Advancing the clock between calls credits tokens at the configured rate.""" + limiter = _make_limiter(tmp_path, fake_clock, rate=4.0, burst=1.0) + + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 # consumes the 1-token burst + fake_clock.now += 0.25 # 0.25s × 4/s = 1 token earned + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 + + +def test_acquire_caps_refill_at_burst(tmp_path, fake_clock): + """A long quiet period must not let the bucket grow past `burst`.""" + limiter = _make_limiter(tmp_path, fake_clock, rate=10.0, burst=2.0) + + fake_clock.now += 1_000 # would earn 10_000 tokens uncapped + # Only `burst` (=2) immediate calls should succeed before throttling. + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 + waited = limiter.acquire(PROVIDER_ANTHROPIC) + assert waited > 0 + + +def test_acquire_independent_buckets_per_provider(tmp_path, fake_clock): + """Anthropic exhaustion must not throttle Azure (each column has its own bucket).""" + limiter = _make_limiter(tmp_path, fake_clock, rate=2.0, burst=1.0) + + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 + # Anthropic bucket is now empty; Azure is untouched. + assert limiter.acquire(PROVIDER_AZURE) == 0.0 + + +def test_acquire_persists_state_across_limiter_instances(tmp_path): + """A fresh RateLimiter must read the on-disk state, not start fresh. + + This is the property that makes the limiter cross-process: an + xdist worker created mid-run sees the credit consumed by other + workers, instead of getting its own private bucket. + """ + cfg = {p: ProviderConfig(rate_per_sec=10.0, burst=2.0) for p in ALL_PROVIDERS} + state = {"now": 1_000.0, "sleeps": []} + + def clock(): + return state["now"] + + def sleep(seconds): + state["sleeps"].append(seconds) + state["now"] += seconds + + first = RateLimiter(config=cfg, state_dir=tmp_path, clock=clock, sleep=sleep) + first.acquire(PROVIDER_ANTHROPIC) + first.acquire(PROVIDER_ANTHROPIC) + # bucket is now empty + + second = RateLimiter(config=cfg, state_dir=tmp_path, clock=clock, sleep=sleep) + waited = second.acquire(PROVIDER_ANTHROPIC) + assert waited > 0 # had to wait, didn't see a fresh full bucket + + +def test_acquire_recovers_from_corrupt_state_file(tmp_path, fake_clock): + """A truncated/garbage state file must not crash the test session.""" + state_file = tmp_path / f"{PROVIDER_ANTHROPIC}.json" + state_file.write_text("not-json {{") + + limiter = _make_limiter(tmp_path, fake_clock, rate=5.0, burst=5.0) + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 + + +def test_acquire_handles_clock_going_backward(tmp_path, fake_clock): + """Across a host suspend/resume the monotonic clock can briefly + go backward; we must not interpret that as removing tokens.""" + limiter = _make_limiter(tmp_path, fake_clock, rate=1.0, burst=2.0) + limiter.acquire(PROVIDER_ANTHROPIC) + fake_clock.now -= 10 # clock moved backward + # Bucket should still have ~1 token left from the burst, not -9. + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 + + +# --------------------------------------------------------------------------- +# Process-default singleton +# --------------------------------------------------------------------------- + + +def test_use_limiter_swaps_default_for_block(tmp_path): + sentinel_cfg = { + p: ProviderConfig(rate_per_sec=0.0, burst=0.0) for p in ALL_PROVIDERS + } + sentinel = RateLimiter(config=sentinel_cfg, state_dir=tmp_path) + reset_default_limiter() + try: + with use_limiter(sentinel): + assert get_default_limiter() is sentinel + # After the context exits, the default goes back to whatever it + # was — in this test that's "rebuilt on next access" because we + # called reset_default_limiter() above. + assert get_default_limiter() is not sentinel + finally: + reset_default_limiter() + + +# --------------------------------------------------------------------------- +# Persistence shape +# --------------------------------------------------------------------------- + + +def test_state_file_is_json_after_acquire(tmp_path, fake_clock): + limiter = _make_limiter(tmp_path, fake_clock, rate=5.0, burst=5.0) + limiter.acquire(PROVIDER_ANTHROPIC) + state_file = tmp_path / f"{PROVIDER_ANTHROPIC}.json" + payload = json.loads(state_file.read_text()) + assert "tokens" in payload + assert "last_refill" in payload + assert payload["tokens"] == pytest.approx(4.0) diff --git a/tests/e2e/claude_code/_pr_gate_unit_tests/__init__.py b/tests/e2e/claude_code/_pr_gate_unit_tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py b/tests/e2e/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py new file mode 100644 index 00000000000..d698131670a --- /dev/null +++ b/tests/e2e/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py @@ -0,0 +1,147 @@ +"""Pin tests for the `Bash`-using compat cells. + +Every cell that passes `--allowed-tools Bash` to the `claude` CLI is +giving a model-controlled response the ability to run host commands. +On the PR-gate CircleCI machine executor, those commands have access +to the Docker socket and can read `docker inspect compat-proxy` to +recover the provider credentials living inside the proxy container. + +To narrow that surface, every Bash-using cell must: + +1. Restrict the allow rule to the *exact* command `Bash(echo pong)` so + a compromised provider response cannot turn `Bash` into arbitrary + host execution by emitting a `tool_use` with a different command. + +2. Pair it with `--permission-mode dontAsk` so anything not matching + an allow rule is auto-denied instead of prompting (which would + abort the CLI in headless mode, but auto-denial is the explicit + contract). + +These restrictions are enforced by the `claude` CLI, not by the +model — see https://code.claude.com/docs/en/permissions for the +permission-rule precedence (`deny` → `ask` → `allow`). + +This test scans every cell under the three Bash-using feature +directories (`tool_use`, `tool_use_streaming`, `thinking_with_tool_use`) +and pins both requirements so a future test refactor cannot silently +revert any cell to the broad `Bash` allow that was originally +flagged by Veria. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Iterable + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[4] +CLAUDE_CODE_DIR = REPO_ROOT / "tests" / "e2e" / "claude_code" + +# Feature directories whose cells drive the `Bash` built-in tool. Add +# new entries here when a new Bash-using feature is added; the test +# fails loudly for any unhandled directory so we never miss one by +# silent omission. +BASH_FEATURE_DIRS = ( + "tool_use", + "tool_use_streaming", + "thinking_with_tool_use", +) + + +def _bash_cells() -> Iterable[Path]: + for feature in BASH_FEATURE_DIRS: + feature_dir = CLAUDE_CODE_DIR / feature + assert feature_dir.is_dir(), ( + f"{feature_dir} is missing — BASH_FEATURE_DIRS is out of sync " + f"with the layout under tests/e2e/claude_code/." + ) + for path in sorted(feature_dir.glob("test_*.py")): + yield path + + +def _has_bare_bash_token(text: str) -> bool: + """Return True if `text` contains a `"Bash"` token outside the + `"Bash(echo pong)"` allow rule. + + Extracted as a pure helper so the negative path can be unit-tested + directly. Without it, the previous structure of this assertion was + `'"Bash"' not in text or '"Bash(echo pong)"' in text`, which + short-circuits to True any time the allow rule is present and lets + a stray bare `"Bash"` slip through the security pin undetected. + """ + return '"Bash"' in text.replace('"Bash(echo pong)"', "") + + +@pytest.mark.parametrize( + "cell", list(_bash_cells()), ids=lambda p: str(p.relative_to(REPO_ROOT)) +) +def test_bash_allow_rule_is_pinned_to_exact_echo_pong(cell: Path) -> None: + """The cell must pass `Bash(echo pong)` as the allow rule, not the + unrestricted `Bash` value that was originally flagged.""" + text = cell.read_text() + assert '"Bash(echo pong)"' in text, ( + f"{cell.relative_to(REPO_ROOT)} must restrict `--allowed-tools` to " + f'`Bash(echo pong)` (exact-match pattern). Unrestricted `"Bash"` ' + f"grants arbitrary host command execution to model-controlled " + f"tool_use blocks, which can read `docker inspect compat-proxy` " + f"to exfiltrate provider credentials from the proxy container." + ) + # The only place `"Bash"` (the bare token, surrounded by quotes + # exactly as it would appear in `--allowed-tools` lists) is allowed + # to appear is *inside* the exact-match `"Bash(echo pong)"` rule. + # `_has_bare_bash_token` keeps that scan independent of the first + # assertion — otherwise `'"Bash"' not in text or '"Bash(echo pong)"' + # in text` short-circuits to True and lets a stray bare `"Bash"` + # slip through silently. + assert not _has_bare_bash_token(text), ( + f"{cell.relative_to(REPO_ROOT)} still references the unrestricted " + f'`"Bash"` value outside the `"Bash(echo pong)"` allow rule — ' + f"sweep it out before merging." + ) + + +def test_has_bare_bash_token_flags_unrestricted_value(): + """A file that allows the bare `"Bash"` token alongside the + exact-match rule must be flagged. Without this guard the security + pin reverts to the dead-code `or` it had originally, which let + arbitrary host commands through under the noise of a passing test. + """ + text = '--allowed-tools "Bash" "Bash(echo pong)"' + assert _has_bare_bash_token(text) + + +def test_has_bare_bash_token_accepts_only_exact_match(): + """The standard pattern — only the exact-match allow rule, no bare + `"Bash"` — must be accepted. This is the shape every Bash-using + cell in the suite is required to take. + """ + text = '--allowed-tools "Bash(echo pong)" --permission-mode "dontAsk"' + assert not _has_bare_bash_token(text) + + +def test_has_bare_bash_token_ignores_unrelated_substrings(): + """`Bash(echo pong)` is the only allowed shape; substrings like + `BashTool` or `Bashing` are unrelated identifiers and must not be + confused with the bare `"Bash"` token (i.e. the exact quoted + string `"Bash"`).""" + text = "BashTool helper used by the bashing harness" + assert not _has_bare_bash_token(text) + + +@pytest.mark.parametrize( + "cell", list(_bash_cells()), ids=lambda p: str(p.relative_to(REPO_ROOT)) +) +def test_bash_cell_uses_dontask_permission_mode(cell: Path) -> None: + """The cell must pair the allow rule with `--permission-mode dontAsk` + so tool calls that don't match the allow rule are auto-denied (as + opposed to defaulting to "ask", which in headless mode would + succeed without ever surfacing the security issue).""" + text = cell.read_text() + assert '"--permission-mode"' in text and '"dontAsk"' in text, ( + f"{cell.relative_to(REPO_ROOT)} must pass `--permission-mode dontAsk` " + f"alongside the `Bash(echo pong)` allow rule. Without dontAsk, " + f"commands outside the allow rule fall back to the default ask-" + f"mode behavior, which in `--print` (headless) mode is non-" + f"interactive — defeating the explicit-allow contract." + ) diff --git a/tests/e2e/claude_code/_pr_gate_unit_tests/test_pr_gate_version_resolver.py b/tests/e2e/claude_code/_pr_gate_unit_tests/test_pr_gate_version_resolver.py new file mode 100644 index 00000000000..5c516da81c5 --- /dev/null +++ b/tests/e2e/claude_code/_pr_gate_unit_tests/test_pr_gate_version_resolver.py @@ -0,0 +1,164 @@ +"""Unit tests for the Claude Code PR-Gate Version Resolver. + +The resolver picks the newest `@anthropic-ai/claude-code` version whose +publish timestamp is at least 3 days old. The 3-day window is a security +review buffer: a malicious or broken Claude Code release that slipped +through the npm publish process gets at least 72 hours to be detected +before it can land in the LiteLLM PR gate. + +The unit tests inject npm metadata directly (no network) and a fixed +`as_of` clock (no real time), so they run anywhere and never flake on +the wall clock or registry availability. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +import pytest + +from claude_code.pr_gate_version_resolver import ( + NoEligibleVersionError, + resolve_pr_gate_version, +) + + +def _t(iso: str) -> str: + """Helper for readable ISO-8601 publish timestamps in fixtures.""" + return iso + + +# A clock fixed at a moment well after every fixture publish time below. +NOW = datetime(2026, 4, 25, 12, 0, 0, tzinfo=timezone.utc) + + +def _metadata_with_times(times: dict) -> dict: + """Shape an npm `packument`-like dict with the `time` field populated. + + The npm registry response includes `time.created` / `time.modified` + keys alongside per-version timestamps; the resolver must skip those. + """ + return { + "name": "@anthropic-ai/claude-code", + "time": { + "created": _t("2024-01-01T00:00:00.000Z"), + "modified": _t("2026-04-25T00:00:00.000Z"), + **times, + }, + } + + +def test_picks_newest_version_at_least_three_days_old(): + metadata = _metadata_with_times( + { + "2.1.118": _t("2026-04-15T10:00:00.000Z"), + "2.1.119": _t("2026-04-21T10:00:00.000Z"), # 4d 2h old + "2.1.120": _t("2026-04-23T10:00:00.000Z"), # 2d 2h old — too new + "2.1.121": _t("2026-04-25T11:00:00.000Z"), # 1h old — too new + } + ) + assert resolve_pr_gate_version(metadata=metadata, as_of=NOW) == "2.1.119" + + +def test_skips_created_and_modified_meta_keys(): + """`time` contains `created` / `modified` non-version entries — must be ignored.""" + metadata = { + "name": "@anthropic-ai/claude-code", + "time": { + "created": _t("2024-01-01T00:00:00.000Z"), + "modified": _t("2026-04-25T00:00:00.000Z"), + "2.0.0": _t("2026-04-10T00:00:00.000Z"), + }, + } + assert resolve_pr_gate_version(metadata=metadata, as_of=NOW) == "2.0.0" + + +def test_min_age_boundary_is_inclusive(): + """A version published exactly 3 days ago is eligible (>= cutoff).""" + three_days_ago = NOW - timedelta(days=3) + metadata = _metadata_with_times( + { + "2.1.0": three_days_ago.isoformat().replace("+00:00", "Z"), + } + ) + assert resolve_pr_gate_version(metadata=metadata, as_of=NOW) == "2.1.0" + + +def test_raises_when_every_version_is_too_new(): + metadata = _metadata_with_times( + { + "2.1.121": _t("2026-04-25T08:00:00.000Z"), # 4h old + "2.1.120": _t("2026-04-24T10:00:00.000Z"), # ~26h old + } + ) + with pytest.raises(NoEligibleVersionError): + resolve_pr_gate_version(metadata=metadata, as_of=NOW) + + +def test_raises_when_metadata_has_no_versions(): + metadata = {"name": "@anthropic-ai/claude-code", "time": {}} + with pytest.raises(NoEligibleVersionError): + resolve_pr_gate_version(metadata=metadata, as_of=NOW) + + +def test_picks_latest_publish_time_not_largest_semver(): + """If a patch is published to an old major after a newer release, + "newest" is by publish time, not semver string ordering.""" + metadata = _metadata_with_times( + { + "1.9.99": _t("2026-04-22T10:00:00.000Z"), # patched recently — wins + "2.0.0": _t("2026-03-01T10:00:00.000Z"), # older publish + } + ) + assert resolve_pr_gate_version(metadata=metadata, as_of=NOW) == "1.9.99" + + +def test_uses_custom_min_age(): + metadata = _metadata_with_times( + { + "1.0.0": _t("2026-04-23T10:00:00.000Z"), # 2d 2h old + "0.9.0": _t("2026-04-10T10:00:00.000Z"), # 15d old + } + ) + # min_age = 5 days disqualifies 1.0.0 + out = resolve_pr_gate_version( + metadata=metadata, as_of=NOW, min_age=timedelta(days=5) + ) + assert out == "0.9.0" + + +def test_excludes_prerelease_versions(): + """Pre-release tags (1.0.0-alpha.1, 2.0.0-rc.1, etc.) must never win, + even if their publish timestamp is the newest eligible one.""" + metadata = _metadata_with_times( + { + "2.1.119": _t("2026-04-21T10:00:00.000Z"), # stable, 4d old + "2.2.0-alpha.1": _t("2026-04-22T10:00:00.000Z"), # newer publish + "2.2.0-rc.1": _t("2026-04-22T11:00:00.000Z"), # newest publish + "3.0.0-beta": _t("2026-04-22T12:00:00.000Z"), # newest publish + } + ) + assert resolve_pr_gate_version(metadata=metadata, as_of=NOW) == "2.1.119" + + +def test_raises_when_only_prereleases_are_eligible(): + metadata = _metadata_with_times( + { + "2.2.0-alpha.1": _t("2026-04-22T10:00:00.000Z"), + "2.2.0-rc.1": _t("2026-04-22T11:00:00.000Z"), + } + ) + with pytest.raises(NoEligibleVersionError): + resolve_pr_gate_version(metadata=metadata, as_of=NOW) + + +def test_resolver_uses_fetcher_when_metadata_not_provided(): + captured = {} + + def fake_fetch(package_name: str) -> dict: + captured["package"] = package_name + return _metadata_with_times({"3.0.0": _t("2026-04-10T10:00:00.000Z")}) + + out = resolve_pr_gate_version(as_of=NOW, fetcher=fake_fetch) + assert out == "3.0.0" + assert captured["package"] == "@anthropic-ai/claude-code" diff --git a/tests/e2e/claude_code/_publisher_unit_tests/__init__.py b/tests/e2e/claude_code/_publisher_unit_tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_pytest_scrubs_env.py b/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_pytest_scrubs_env.py new file mode 100644 index 00000000000..418d308674b --- /dev/null +++ b/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_pytest_scrubs_env.py @@ -0,0 +1,95 @@ +"""Pin: the cron `pytest` invocation must run under `env -i`. + +The systemd service `litellm-compat-matrix.service` loads provider +credentials (`ANTHROPIC_API_KEY`, `AWS_BEARER_TOKEN_BEDROCK`, +`AZURE_FOUNDRY_API_KEY`, `VERTEXAI_*`) and the agent-shin GitHub token +(`AGENT_SHIN_GITHUB_TOKEN`) into `run_daily.sh`'s environment from +`/etc/litellm-compat-matrix.env`. Pytest only needs to talk to the +loopback proxy at `127.0.0.1:${PROXY_PORT}` and has no legitimate reason +to see provider creds in its own `os.environ`. Leaving them in would +let a test under `tests/e2e/claude_code/` read them via `os.environ` and +exfiltrate them, and would also let a model-directed `Read` tool call +during a PDF/vision cell reach `/proc//environ`. The +PR-gate's pytest step in `.circleci/config.yml` already runs under +`env -i`; this pin enforces the same scrub on the cron path. +""" + +from __future__ import annotations + +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[4] +RUN_DAILY = REPO_ROOT / "tests" / "e2e" / "claude_code" / "cron_vm" / "run_daily.sh" + + +def _pytest_invocation_block() -> str: + """Return only the executable lines around the pytest invocation. + + Comment text in run_daily.sh explains *why* certain credential + names must not appear, so a naïve substring scan over the whole + region would false-positive on the rationale itself. Strip lines + whose first non-space character is `#`. + """ + body = RUN_DAILY.read_text() + start = body.index('log "running pytest"') + end = body.index("PYTEST_EXIT=$?", start) + return "\n".join( + line for line in body[start:end].splitlines() + if line.lstrip()[:1] != "#" + ) + + +def test_pytest_invocation_wraps_in_env_i() -> None: + block = _pytest_invocation_block() + assert "env -i" in block, ( + "run_daily.sh: the pytest invocation must run under `env -i` so " + "PR-controlled test code under tests/e2e/claude_code/ cannot read " + "provider/agent-shin credentials out of the systemd service " + "environment, and so a model-directed `Read` tool call cannot " + "reach /proc//environ to pull them out." + ) + assert block.index("env -i") < block.index('"${WORKTREE_UV}" run pytest'), ( + "run_daily.sh: `env -i` must precede the pytest invocation; " + "otherwise pytest inherits the full credential-bearing env." + ) + + +def test_pytest_invocation_env_i_excludes_provider_secrets() -> None: + block = _pytest_invocation_block() + for forbidden in ( + "ANTHROPIC_API_KEY", + "AWS_BEARER_TOKEN_BEDROCK", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "VERTEXAI_CREDENTIALS", + "VERTEXAI_PROJECT", + "VERTEXAI_LOCATION", + "AZURE_FOUNDRY_API_KEY", + "AZURE_FOUNDRY_API_BASE", + "GITHUB_TOKEN", + "AGENT_SHIN_GITHUB_TOKEN", + ): + assert forbidden not in block, ( + f"run_daily.sh: the pytest-step `env -i` allowlist must not " + f"pass {forbidden} through. Found it inside the pytest " + f"invocation block." + ) + + +def test_pytest_invocation_passes_proxy_url_and_key_explicitly() -> None: + block = _pytest_invocation_block() + assert "LITELLM_PROXY_BASE_URL=" in block, ( + "run_daily.sh: the pytest `env -i` block must still pass " + "LITELLM_PROXY_BASE_URL so the test suite knows where to find " + "the loopback proxy." + ) + assert "LITELLM_PROXY_API_KEY=" in block, ( + "run_daily.sh: the pytest `env -i` block must still pass " + "LITELLM_PROXY_API_KEY so the test suite can authenticate to " + "the loopback proxy." + ) + assert "COMPAT_RESULTS_PATH=" in block, ( + "run_daily.sh: the pytest `env -i` block must still pass " + "COMPAT_RESULTS_PATH so the conftest writes the per-cell " + "tagged-union artifact to the script-managed path." + ) diff --git a/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_release_pagination.py b/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_release_pagination.py new file mode 100644 index 00000000000..fc733845ba3 --- /dev/null +++ b/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_release_pagination.py @@ -0,0 +1,289 @@ +"""Regression tests for the GitHub release pagination in `run_daily.sh`. + +The cron job resolves "newest LiteLLM v*-stable" via the GitHub Releases +API. A previous version of the loop broke as soon as the current page +contained ANY v*-stable tag. The Releases endpoint orders by +`created_at`, NOT by semver, so a backport on an older series cut today +(e.g. v1.80.1-stable) can land on an earlier page than a higher-version +release cut two weeks ago (e.g. v1.83.0-stable). The early-break would +silently pin the cron to a stale tag because the higher-version release +on a later page never made it into the merged set the final `sort_by` +consumed. + +These tests pin two things: + + 1. The buggy early-break-on-first-stable pattern must not return. + 2. The loop still terminates early on the standard "empty page" guard + so a quiet release feed doesn't burn API quota. + +The shell loop itself is exercised end-to-end with a fake `curl` that +serves canned page JSON, demonstrating that the resolved tag is the +highest-semver stable across all pages even when the highest tag lives +on page 2+. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import textwrap +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[4] +RUN_DAILY = REPO_ROOT / "tests" / "e2e" / "claude_code" / "cron_vm" / "run_daily.sh" + +# The extracted snippet starts AFTER `log`/`die` are defined in run_daily.sh, +# so the test harness has to provide its own stubs. Without them, a failure +# inside the snippet (e.g. jq returning an empty LITELLM_VERSION) would crash +# with `bash: die: command not found` (exit 127) instead of the intended +# diagnostic, making test failures unnecessarily hard to debug. +_PREAMBLE = ( + "set -Eeuo pipefail\n" + "log() { printf '==> %s\\n' \"$*\" >&2; }\n" + "die() { printf 'ERROR: %s\\n' \"$*\" >&2; exit 1; }\n" +) + + +def test_run_daily_does_not_early_break_on_first_stable_page() -> None: + """The regex pattern `select(test("...stable$"))] | length > 0` followed + by `break` is exactly the buggy early-stop. If it ever returns the + cron will silently start testing against a stale stable tag. + """ + body = RUN_DAILY.read_text() + assert ( + "length > 0" not in body + or "break" not in body + or ( + # If both substrings exist, make sure they aren't both inside the + # same release-pagination loop. The current loop only contains + # a `break` for the empty-page guard, not for any "length > 0" + # condition. + not _shares_loop_body(body, "length > 0", "break") + ) + ), ( + "run_daily.sh contains the old early-break-on-stable pattern. The " + "Releases endpoint orders by created_at, not semver, so breaking " + "on first-stable-seen can miss higher-versioned releases sitting " + "on later pages." + ) + + +def _shares_loop_body(body: str, needle_a: str, needle_b: str) -> bool: + """Heuristic: do both needles live inside a `for page in ...; do ... done` + block? Used as a defensive guard for the static check above.""" + in_loop = False + saw_a = False + saw_b = False + for line in body.splitlines(): + stripped = line.strip() + if stripped.startswith("for page in"): + in_loop = True + saw_a = False + saw_b = False + continue + if in_loop and stripped == "done": + if saw_a and saw_b: + return True + in_loop = False + continue + if in_loop: + if needle_a in line: + saw_a = True + if needle_b in line: + saw_b = True + return False + + +def test_run_daily_keeps_empty_page_break_guard() -> None: + """The empty-page break is the only break that should remain in the + pagination loop — without it a quiet release feed wastes API quota + walking past the last real page.""" + body = RUN_DAILY.read_text() + assert "jq 'length' \"${PAGE_JSON}\"" in body, ( + "run_daily.sh must still detect empty pages via `jq 'length' " + "${PAGE_JSON}`; without this the loop walks the full 5-page cap " + "even when there are no more releases." + ) + assert ( + '== "0"' in body + ), 'The empty-page guard must compare jq\'s length output to "0".' + + +def _make_fake_curl(scratch: Path, pages: dict[int, str]) -> Path: + """Build a fake `curl` shim that serves the canned page JSON for + each `page=N` request and an empty array for any page past the + last canned one. + + The shim mimics just enough of curl's CLI surface for the cron + script: it accepts the headers + URL we pass, ignores everything + we don't need, and writes the canned body to either stdout or the + --output target if one is given. + """ + pages_dir = scratch / "pages" + pages_dir.mkdir() + for page_num, body in pages.items(): + (pages_dir / f"page{page_num}.json").write_text(body) + + curl_path = scratch / "curl" + curl_path.write_text( + textwrap.dedent( + f"""\ + #!/usr/bin/env bash + # Fake curl for run_daily.sh release pagination tests. Serves + # page JSON from {pages_dir} keyed by the `page=` query value, + # and returns "[]" for pages past the last canned one (which + # is exactly how the real GitHub API behaves past the end). + url="" + output="" + while [[ $# -gt 0 ]]; do + case "$1" in + -fsS|-fsSL|-H|-o|--output) + if [[ "$1" == "-o" || "$1" == "--output" ]]; then + output="$2"; shift 2 + elif [[ "$1" == "-H" ]]; then + shift 2 + else + shift + fi + ;; + http*) + url="$1"; shift + ;; + *) + shift + ;; + esac + done + page="$(printf '%s' "$url" | sed -n 's/.*[?&]page=\\([0-9]*\\).*/\\1/p')" + [[ -z "$page" ]] && page=1 + file="{pages_dir}/page${{page}}.json" + if [[ -f "$file" ]]; then + if [[ -n "$output" ]]; then cp "$file" "$output"; else cat "$file"; fi + else + if [[ -n "$output" ]]; then printf '[]' > "$output"; else printf '[]'; fi + fi + """ + ) + ) + curl_path.chmod(0o755) + return curl_path + + +def _extract_resolution_snippet() -> str: + """Pull the pagination + sort_by + assignment block out of run_daily.sh + so the test exercises the actual production code path (not a copy). + + The block is everything from the GH_AUTH_HEADER setup down through + the LITELLM_VERSION emission. + """ + body = RUN_DAILY.read_text() + start = body.index("GH_AUTH_HEADER=()") + end = body.index('log "resolved litellm:') + return body[start:end] + + +@pytest.mark.skipif(shutil.which("jq") is None, reason="jq not available") +def test_run_daily_resolves_highest_semver_across_pages(tmp_path: Path) -> None: + """End-to-end: drive the actual run_daily.sh pagination loop with a + fake curl whose page 1 contains a freshly-cut LOW-version backport + (v1.80.1-stable) and page 2 contains a two-weeks-old HIGH-version + release (v1.83.0-stable). The correct behavior is to resolve + v1.83.0-stable. The pre-fix behavior would resolve v1.80.1-stable + because the early-break consumed only page 1. + """ + pages = { + # Page 1: most-recently-created releases. The order here matches + # what /releases?page=1 returns: created-at descending. The + # freshly-cut v1.80.1-stable backport sits at the top, plus a + # bunch of non-stable releases. + 1: """[ + {"tag_name": "v1.84.0-nightly.1"}, + {"tag_name": "v1.80.1-stable"}, + {"tag_name": "v1.84.0-nightly.0"} + ]""", + # Page 2: older releases. The HIGHER-version stable lives here + # because it was cut two weeks ago, before the v1.80.1 backport. + 2: """[ + {"tag_name": "v1.83.0-rc.5"}, + {"tag_name": "v1.83.0-stable"}, + {"tag_name": "v1.82.4-stable"} + ]""", + # Page 3+: empty -> the loop's empty-page guard fires here. + } + fake_curl_dir = tmp_path / "shim" + fake_curl_dir.mkdir() + _make_fake_curl(fake_curl_dir, pages) + + workdir = tmp_path / "work" + workdir.mkdir() + + snippet = _extract_resolution_snippet() + script = ( + _PREAMBLE + + f"WORKDIR={workdir!s}\n" + + snippet + + 'printf "%s" "${LITELLM_VERSION}"\n' + ) + + env = { + **os.environ, + "PATH": f"{fake_curl_dir}:{os.environ.get('PATH', '')}", + } + # Make sure the loop hits the fake curl, not the system one. + env.pop("GITHUB_TOKEN", None) + result = subprocess.run( + ["bash", "-c", script], + capture_output=True, + text=True, + env=env, + check=True, + ) + assert result.stdout == "v1.83.0-stable", ( + f"Expected the highest-semver stable across pages 1-2, got " + f"{result.stdout!r}. stderr={result.stderr!r}" + ) + + +@pytest.mark.skipif(shutil.which("jq") is None, reason="jq not available") +def test_run_daily_terminates_on_empty_page(tmp_path: Path) -> None: + """The empty-page guard must fire so we don't always walk all 5 + pages. With a single populated page and an empty page 2 we should + stop after fetching page 2 (the first empty response).""" + pages = {1: '[{"tag_name": "v1.50.0-stable"}]'} + fake_curl_dir = tmp_path / "shim" + fake_curl_dir.mkdir() + _make_fake_curl(fake_curl_dir, pages) + + workdir = tmp_path / "work" + workdir.mkdir() + + snippet = _extract_resolution_snippet() + script = ( + _PREAMBLE + + f"WORKDIR={workdir!s}\n" + + snippet + + 'printf "%s" "${LITELLM_VERSION}"\n' + ) + + env = { + **os.environ, + "PATH": f"{tmp_path}/shim:{os.environ.get('PATH', '')}", + } + env.pop("GITHUB_TOKEN", None) + result = subprocess.run( + ["bash", "-c", script], + capture_output=True, + text=True, + env=env, + check=True, + ) + assert result.stdout == "v1.50.0-stable" + # Only pages 1 and 2 should have been fetched (2 is empty -> break). + assert (workdir / "releases.page2.json").exists() + assert not (workdir / "releases.page3.json").exists(), ( + "Empty-page guard didn't fire — the loop kept walking past the " + "first empty response." + ) diff --git a/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_version_probe_scrubs_env.py b/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_version_probe_scrubs_env.py new file mode 100644 index 00000000000..1c3959764f3 --- /dev/null +++ b/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_version_probe_scrubs_env.py @@ -0,0 +1,92 @@ +"""Pin: the cron `claude --version` probe must run under `env -i`. + +The systemd service `litellm-compat-matrix.service` loads provider +credentials (`ANTHROPIC_API_KEY`, `AWS_BEARER_TOKEN_BEDROCK`, +`AZURE_FOUNDRY_API_KEY`) and the agent-shin GitHub token +(`AGENT_SHIN_GITHUB_TOKEN`) into `run_daily.sh`'s environment from +`/etc/litellm-compat-matrix.env`. Running the npm-installed `claude` +binary directly there would hand that full env to package code, so a +compromised `@anthropic-ai/claude-code` release could read those +secrets out of `os.environ` before the proxy or test harness ever +starts. The version probe must be wrapped in `env -i` with a minimal +PATH/HOME/USER/TERM/LANG/LC_ALL/TMPDIR allowlist — matching the +PR-gate's resolver/npm-install/pytest scrubs. +""" + +from __future__ import annotations + +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[4] +RUN_DAILY = REPO_ROOT / "tests" / "e2e" / "claude_code" / "cron_vm" / "run_daily.sh" + + +def _version_probe_block() -> str: + body = RUN_DAILY.read_text() + start = body.index("CLAUDE_CODE_VERSION=") + end = body.index('[[ -n "${CLAUDE_CODE_VERSION}" ]]', start) + return body[start:end] + + +def test_version_probe_wraps_claude_in_env_i() -> None: + block = _version_probe_block() + assert "env -i" in block, ( + "run_daily.sh: the `claude --version` probe must run under " + "`env -i` so a compromised @anthropic-ai/claude-code package " + "cannot read provider/GitHub credentials out of the systemd " + "service environment." + ) + assert block.index("env -i") < block.index("claude --version"), ( + "run_daily.sh: `env -i` must precede `claude --version`; " + "otherwise the binary inherits the full credential-bearing env." + ) + + +def test_version_probe_env_i_excludes_provider_secrets() -> None: + block = _version_probe_block() + for forbidden in ( + "ANTHROPIC_API_KEY", + "AWS_BEARER_TOKEN_BEDROCK", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "VERTEXAI_CREDENTIALS", + "AZURE_FOUNDRY_API_KEY", + "GITHUB_TOKEN", + "AGENT_SHIN_GITHUB_TOKEN", + ): + assert forbidden not in block, ( + f"run_daily.sh: the version-probe `env -i` allowlist must " + f"not pass {forbidden} through. Found it inside the probe " + f"block." + ) + + +def test_version_probe_uses_isolated_home_not_runtime_user_home() -> None: + """Pin: the `claude --version` probe runs under a fresh empty HOME. + + `ProtectHome=read-only` in the systemd unit allows reads of the + runtime user's real home directory. If the probe's `env -i` + block forwards `HOME=${HOME}`, a compromised `claude` package + can `os.path.expanduser("~/.config/gh/hosts.yml")` or + `os.path.expanduser("~/.ssh/...")` and exfiltrate the contents + before the proxy or test harness ever starts. The probe must + point HOME at a per-run tmpdir under `${WORKDIR}` so the CLI + sees an empty HOME instead. + """ + block = _version_probe_block() + body = RUN_DAILY.read_text() + + assert "CLAUDE_PROBE_HOME=" in body, ( + "run_daily.sh: must define a `CLAUDE_PROBE_HOME` per-run tmpdir " + "for the `claude --version` probe so the CLI never sees the " + "runtime user's real $HOME." + ) + assert 'HOME="${CLAUDE_PROBE_HOME}"' in block, ( + "run_daily.sh: the probe's `env -i` block must set HOME to " + "the per-run isolated tmpdir, not to the runtime user's $HOME." + ) + assert 'HOME="${HOME}"' not in block, ( + "run_daily.sh: the probe's `env -i` block must not forward the " + "runtime user's $HOME to `claude --version`. Use the isolated " + "$CLAUDE_PROBE_HOME tmpdir instead." + ) diff --git a/tests/e2e/claude_code/_publisher_unit_tests/test_systemd_unit_credential_isolation.py b/tests/e2e/claude_code/_publisher_unit_tests/test_systemd_unit_credential_isolation.py new file mode 100644 index 00000000000..12edce3cb50 --- /dev/null +++ b/tests/e2e/claude_code/_publisher_unit_tests/test_systemd_unit_credential_isolation.py @@ -0,0 +1,104 @@ +"""Pin: the cron systemd unit hides credential-bearing dotdirs. + +`ProtectHome=read-only` blocks writes to /home/mateo but still allows +reads. A model-directed `Read` tool call (the PDF cells pass +`--allowed-tools Read` to the `claude` CLI) or a compromised +`@anthropic-ai/claude-code` package can read absolute paths under +the runtime user's home and exfiltrate the contents — even with the +per-`claude`-invocation HOME isolation in place, because absolute +paths bypass `~`-expansion. + +This file pins the second line of defense: the systemd unit lists +the credential-bearing dotdirs (`~/.config/gh`, `~/.ssh`, `~/.aws`, +`~/.docker`, `~/.kube`, `~/.gnupg`) under `InaccessiblePaths=` so +the kernel hides them from every process in the unit's mount +namespace, including any child of `claude --version` or the pytest +run. It also pins that `~/.config/gh` is *not* in `ReadWritePaths=` +— we pass `GH_TOKEN` inline to every `gh` invocation in +`run_daily.sh`, so the host gh-cli config is unused. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[4] +SERVICE = ( + REPO_ROOT / "tests" / "e2e" / "claude_code" / "cron_vm" / "litellm-compat-matrix.service" +) + + +def _service_text() -> str: + return SERVICE.read_text() + + +def _directive(name: str) -> str: + """Return the value of a single-line systemd directive (or empty).""" + text = _service_text() + match = re.search(rf"^\s*{re.escape(name)}\s*=\s*(.*)$", text, re.MULTILINE) + return match.group(1).strip() if match else "" + + +def test_inaccessible_paths_hides_credential_dotdirs() -> None: + """Every credential-bearing dotdir must be under `InaccessiblePaths=`.""" + inaccessible = _directive("InaccessiblePaths") + assert inaccessible, ( + "litellm-compat-matrix.service: must declare `InaccessiblePaths=` " + "to hide credential dotdirs from the `claude` subprocess and the " + "model-directed Read tool. Without this, an absolute-path read " + "like `Read('/home/mateo/.config/gh/hosts.yml')` exfiltrates " + "the gh-cli token despite the per-invocation HOME isolation." + ) + for path in ( + "/home/mateo/.config/gh", + "/home/mateo/.ssh", + "/home/mateo/.aws", + "/home/mateo/.docker", + "/home/mateo/.kube", + "/home/mateo/.gnupg", + ): + # Tolerated `-` prefix means "ignore if missing on host". + assert path in inaccessible, ( + f"litellm-compat-matrix.service: `{path}` must appear in " + f"`InaccessiblePaths=` so the cron `claude` subprocess can " + f"never read it (even via an absolute path that bypasses " + f"the per-invocation HOME override)." + ) + + +def test_gh_config_is_not_writeable() -> None: + """`~/.config/gh` is not whitelisted under `ReadWritePaths=`. + + We pass `GH_TOKEN` inline to every `gh` invocation in + `run_daily.sh` (`gh repo clone`, `gh pr create`, `gh pr edit`). + The host `~/.config/gh/hosts.yml` is therefore never consulted + or written to. Keeping it out of `ReadWritePaths=` is the second + line of defense: a future regression that drops the inline-token + convention will fail loudly (gh writes a new login config and + hits a read-only filesystem) rather than silently re-introduce + the credential exfiltration surface that + `InaccessiblePaths=/home/mateo/.config/gh` is closing. + """ + rw = _directive("ReadWritePaths") + assert ".config/gh" not in rw, ( + "litellm-compat-matrix.service: `/home/mateo/.config/gh` must " + "*not* appear in `ReadWritePaths=`. We pass `GH_TOKEN` inline " + "to every `gh` invocation in run_daily.sh, so the host gh-cli " + "config is never consulted or written to. Keeping the path out " + "of ReadWritePaths means a future regression that drops the " + "inline-token convention will fail loudly instead of silently " + "re-opening the credential exfiltration surface that " + "`InaccessiblePaths=` is closing." + ) + + +def test_protect_home_is_read_only_or_stricter() -> None: + """`ProtectHome=` must be at least `read-only`.""" + value = _directive("ProtectHome") + assert value in ("read-only", "tmpfs", "yes", "true"), ( + f"litellm-compat-matrix.service: `ProtectHome=` must be `read-only`, " + f"`tmpfs`, or `yes`. Got: {value!r}. Without this, the unit can " + f"write anywhere under /home/mateo, including overwriting " + f"~/.config/gh/hosts.yml." + ) diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/__init__.py b/tests/e2e/claude_code/basic_messaging_non_streaming/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py new file mode 100644 index 00000000000..c06fff28d2d --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py @@ -0,0 +1,47 @@ +"""basic_messaging_non_streaming × Anthropic. + +The thinnest end-to-end path through every layer of the matrix: drive the +real `claude` CLI in headless mode against a running LiteLLM proxy that +routes to Anthropic, and report the outcome via `compat_result`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider + +Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus +4.7; the cell only goes green if all three pass. The shared +`run_basic_messaging_cell` helper fans the three model runs out in +parallel and reports one `compat_result.add(...)` entry per model so +the matrix builder still sees three rows for this (feature, provider). +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell + +# Per the PRD: each cell is exercised against three Claude tiers via the +# Anthropic provider. Aliases are configured in the LiteLLM proxy's +# routing config; the driver only sends the alias. +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + + +def test_basic_messaging_non_streaming_anthropic(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a reply. + + "Basic messaging" means: send a single user prompt, receive any + non-empty assistant text reply, no tools, no streaming, no thinking. + The whole point of this slice is to prove the path works at all — + so the assertion is intentionally lenient on the reply contents. + """ + run_basic_messaging_cell( + compat_result=compat_result, + models=ANTHROPIC_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + ) diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py new file mode 100644 index 00000000000..2a962b244a8 --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py @@ -0,0 +1,53 @@ +"""basic_messaging_non_streaming x Azure (Microsoft Foundry). + +Drive the real `claude` CLI in headless mode against a running LiteLLM +proxy that routes Claude requests to Anthropic's models hosted in +Microsoft Foundry on Azure, and report the outcome via `compat_result`. + +Anthropic announced Claude Haiku 4.5, Sonnet 4.5/4.6, and Opus 4.1/4.6/4.7 +in Microsoft Foundry on 2025-11-18; LiteLLM exposes them via the +`azure_ai/claude-*` provider prefix, which talks to Foundry's +Anthropic-shape `/anthropic/v1/messages` endpoint. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^ + feature_id provider + +Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus +4.7; the cell only goes green if all three pass. The shared +`run_basic_messaging_cell` helper fans the three model runs out in +parallel and reports one `compat_result.add(...)` entry per model so +the matrix builder still sees three rows for this (feature, provider). +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell + +# Per-model aliases registered in the LiteLLM proxy's routing config to +# point at Microsoft Foundry's Anthropic deployments. The driver only +# sends the alias; the proxy is the one that knows the upstream Foundry +# resource URL and API key. +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + + +def test_basic_messaging_non_streaming_azure(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a reply. + + "Basic messaging" means: send a single user prompt, receive any + non-empty assistant text reply, no tools, no streaming, no thinking. + The whole point of this slice is to prove the path works at all — + so the assertion is intentionally lenient on the reply contents. + """ + run_basic_messaging_cell( + compat_result=compat_result, + models=AZURE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + ) diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py new file mode 100644 index 00000000000..2245ed7417a --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py @@ -0,0 +1,42 @@ +"""basic_messaging_non_streaming x Bedrock (Converse). + +Drive the real `claude` CLI in headless mode against a running LiteLLM +proxy that routes Claude requests to AWS Bedrock via the unified +`Converse` API path, and report the outcome via `compat_result`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider + +Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus +4.7; the cell only goes green if all three pass. The shared +`run_basic_messaging_cell` helper fans the three model runs out in +parallel and reports one `compat_result.add(...)` entry per model so +the matrix builder still sees three rows for this (feature, provider). +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell + +# Per-model aliases registered in the LiteLLM proxy's routing config to +# point at Bedrock's Converse endpoint. The driver only sends the alias; +# the proxy is the one that knows the upstream model id and routing +# strategy. +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + + +def test_basic_messaging_non_streaming_bedrock_converse(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" + run_basic_messaging_cell( + compat_result=compat_result, + models=BEDROCK_CONVERSE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + ) diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py new file mode 100644 index 00000000000..e0a6e77f3c1 --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py @@ -0,0 +1,42 @@ +"""basic_messaging_non_streaming x Bedrock (Invoke). + +Drive the real `claude` CLI in headless mode against a running LiteLLM +proxy that routes Claude requests to AWS Bedrock via the legacy +`InvokeModel` API path, and report the outcome via `compat_result`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider + +Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus +4.7; the cell only goes green if all three pass. The shared +`run_basic_messaging_cell` helper fans the three model runs out in +parallel and reports one `compat_result.add(...)` entry per model so +the matrix builder still sees three rows for this (feature, provider). +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell + +# Per-model aliases registered in the LiteLLM proxy's routing config to +# point at Bedrock's legacy InvokeModel endpoint. The driver only sends +# the alias; the proxy is the one that knows the upstream model id and +# routing strategy. +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + + +def test_basic_messaging_non_streaming_bedrock_invoke(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" + run_basic_messaging_cell( + compat_result=compat_result, + models=BEDROCK_INVOKE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + ) diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py new file mode 100644 index 00000000000..e4e2a39e6cd --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py @@ -0,0 +1,42 @@ +"""basic_messaging_non_streaming x Vertex AI. + +Drive the real `claude` CLI in headless mode against a running LiteLLM +proxy that routes Claude requests to Anthropic's models on Google Cloud +Vertex AI, and report the outcome via `compat_result`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider + +Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus +4.7; the cell only goes green if all three pass. The shared +`run_basic_messaging_cell` helper fans the three model runs out in +parallel and reports one `compat_result.add(...)` entry per model so +the matrix builder still sees three rows for this (feature, provider). +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell + +# Per-model aliases registered in the LiteLLM proxy's routing config to +# point at Vertex AI's Anthropic model endpoints. The driver only sends +# the alias; the proxy is the one that knows the upstream publisher +# model id and the GCP region. +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + + +def test_basic_messaging_non_streaming_vertex_ai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" + run_basic_messaging_cell( + compat_result=compat_result, + models=VERTEX_AI_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + ) diff --git a/tests/e2e/claude_code/basic_messaging_streaming/__init__.py b/tests/e2e/claude_code/basic_messaging_streaming/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py b/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py new file mode 100644 index 00000000000..56e3fb6c181 --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py @@ -0,0 +1,46 @@ +"""basic_messaging_streaming x Anthropic. + +Drive the real `claude` CLI in headless `--output-format stream-json +--include-partial-messages` mode against a running LiteLLM proxy that +routes to Anthropic, and report the outcome via `compat_result`. + +The cell goes green only when every Claude tier (a) returns a non-empty +reply and (b) the proxy actually streamed it — i.e. the CLI observed +multiple `stream_event` records carrying raw SSE deltas. A proxy that +buffers the upstream stream and returns a single non-streaming chunk +emits zero such records, which is the regression this row catches. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider + +The shared `run_basic_messaging_cell` helper fans the three Claude tiers +out in parallel inside this single test, with one +`compat_result.add(...)` entry per model so the matrix builder still +sees three rows for this (feature, provider). +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + + +def test_basic_messaging_streaming_anthropic(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty streamed reply (one row per Claude tier). + """ + run_basic_messaging_cell( + compat_result=compat_result, + models=ANTHROPIC_MODELS, + prompt="Count from 1 to 5, one number per line.", + verify_streaming=True, + ) diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py b/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py new file mode 100644 index 00000000000..b6c002d0b27 --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py @@ -0,0 +1,40 @@ +"""basic_messaging_streaming x Azure (Microsoft Foundry). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Claude requests to +Anthropic's models hosted in Microsoft Foundry on Azure, and report the +outcome via `compat_result`. + +Foundry exposes Claude on an Anthropic-shape `/anthropic/v1/messages` +endpoint with native SSE streaming; LiteLLM forwards stream events +through the `azure_ai/claude-*` provider unchanged. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_streaming/test_azure.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + + +def test_basic_messaging_streaming_azure(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty streamed reply (one row per Claude tier). + """ + run_basic_messaging_cell( + compat_result=compat_result, + models=AZURE_MODELS, + prompt="Count from 1 to 5, one number per line.", + verify_streaming=True, + ) diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py new file mode 100644 index 00000000000..44ac54515f0 --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py @@ -0,0 +1,36 @@ +"""basic_messaging_streaming x Bedrock (Converse). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Claude requests to AWS +Bedrock via the unified `Converse` API path, and report the outcome via +`compat_result`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + + +def test_basic_messaging_streaming_bedrock_converse(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty streamed reply (one row per Claude tier). + """ + run_basic_messaging_cell( + compat_result=compat_result, + models=BEDROCK_CONVERSE_MODELS, + prompt="Count from 1 to 5, one number per line.", + verify_streaming=True, + ) diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py new file mode 100644 index 00000000000..1d59d16cdc1 --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py @@ -0,0 +1,36 @@ +"""basic_messaging_streaming x Bedrock (Invoke). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Claude requests to AWS +Bedrock via the legacy `InvokeModel` API path, and report the outcome +via `compat_result`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + + +def test_basic_messaging_streaming_bedrock_invoke(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty streamed reply (one row per Claude tier). + """ + run_basic_messaging_cell( + compat_result=compat_result, + models=BEDROCK_INVOKE_MODELS, + prompt="Count from 1 to 5, one number per line.", + verify_streaming=True, + ) diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py new file mode 100644 index 00000000000..014a31160a8 --- /dev/null +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py @@ -0,0 +1,36 @@ +"""basic_messaging_streaming x Vertex AI. + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Claude requests to +Anthropic's models on Google Cloud Vertex AI, and report the outcome +via `compat_result`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +from claude_code._basic_messaging import run_basic_messaging_cell + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + + +def test_basic_messaging_streaming_vertex_ai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty streamed reply (one row per Claude tier). + """ + run_basic_messaging_cell( + compat_result=compat_result, + models=VERTEX_AI_MODELS, + prompt="Count from 1 to 5, one number per line.", + verify_streaming=True, + ) diff --git a/tests/e2e/claude_code/cli_driver.py b/tests/e2e/claude_code/cli_driver.py new file mode 100644 index 00000000000..97eaa0e6847 --- /dev/null +++ b/tests/e2e/claude_code/cli_driver.py @@ -0,0 +1,542 @@ +"""Claude Code CLI Driver. + +A thin wrapper around the `claude` CLI in headless mode. Every compatibility +test consumes only this module — tests must never shell out directly. This +keeps the subprocess assembly, stream-JSON parsing, and result shape in a +single place that can be unit-tested with a mocked subprocess. + +The driver is deliberately small: it knows how to invoke the CLI, drain its +stream-JSON output, and return a structured `DriverResult`. Higher-level +matrix concerns (status aggregation, manifest lookup, JSON serialization) +live in `matrix_builder.py`. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Union + +from claude_code.rate_limiter import ( + RateLimiter, + get_default_limiter, + infer_provider, +) + +CLAUDE_CLI_DEFAULT = "claude" +# 120s is fine for a single isolated CLI call against an unloaded +# upstream, but the matrix run launches up to 75 concurrent calls and +# upstreams can take several minutes to respond under that contention. +# We expose the timeout as an env var so binary-search runs can grow +# it without touching the test code. +DEFAULT_TIMEOUT_SECONDS = float( + os.environ.get("LITELLM_COMPAT_CLI_TIMEOUT_SECONDS") or 120 +) + +# Env vars the `claude` Node CLI legitimately needs to function: +# locating its own binary + node, basic locale/terminal plumbing. +# Deliberately excludes every credential-bearing var that the +# surrounding CI job sets for the proxy (ANTHROPIC_API_KEY, AWS_*, +# AZURE_*, VERTEXAI_CREDENTIALS, GITHUB_TOKEN, OPENAI_API_KEY, +# DATABASE_URL, ...). The CLI talks to the proxy via the explicit +# ANTHROPIC_BASE_URL/ANTHROPIC_AUTH_TOKEN we set below — it has no +# business reading the proxy's upstream credentials, and a +# compromised CLI release shouldn't be able to exfiltrate them out +# of the CI environment. +# +# `HOME` is intentionally NOT in this list: see `_make_isolated_home` +# below. We give the CLI a fresh empty per-invocation HOME so a +# compromised claude package or a model-directed `Read` tool call +# can't reach files like `~/.config/gh/hosts.yml`, `~/.ssh/id_rsa`, +# or `~/.bash_history` on the cron VM (and on the CircleCI executor +# the same isolation prevents accidentally exposing checkout-adjacent +# files even though the runner home is ephemeral there). +_CLI_ENV_ALLOWLIST: tuple = ( + "PATH", + "USER", + "LOGNAME", + "SHELL", + "TERM", + "TMPDIR", + "LANG", + "LC_ALL", + "LC_CTYPE", + "NODE_PATH", + "NVM_DIR", + "NVM_BIN", +) + + +def _make_isolated_home() -> str: + """Create a fresh empty HOME directory for a single `claude` subprocess. + + The CLI needs *a* writable HOME (it caches per-session state under + `$HOME/.claude/projects//`), but it has no legitimate need + for the *user's* HOME. Handing it the real one means a compromised + `@anthropic-ai/claude-code` release, or a model-directed `Read` + tool call during a PDF/vision cell, can read host files like + `~/.config/gh/hosts.yml` (GitHub CLI host token), `~/.ssh/`, + `~/.bash_history`, or any other dotfile under the runtime user's + home. On the cron VM the runtime user is a real interactive + account (`mateo`) with a populated home directory, so this is a + real exfiltration surface. + + The directory is created under `tempfile.gettempdir()` (which is + `/tmp` on Linux; under systemd's `PrivateTmp=true` that's a + per-service tmpfs that the service user can't otherwise reach). + Caller is responsible for `shutil.rmtree`-ing it after the + subprocess exits. + """ + return tempfile.mkdtemp(prefix="claude-cli-home-") + + +class ClaudeCLIError(RuntimeError): + """Raised when the `claude` CLI cannot be invoked or returns a fatal error.""" + + +@dataclass +class DriverResult: + """Structured outcome of a single `claude` CLI invocation. + + `text` is the assistant's final user-visible reply (joined across any + intermediate `assistant` events for non-streaming runs). `events` is the + raw list of stream-JSON objects emitted by the CLI, preserved so test + authors can write feature-specific assertions (tool calls, cache hits, + usage) without re-parsing stdout. + """ + + text: str + events: List[Dict[str, Any]] = field(default_factory=list) + exit_code: int = 0 + stderr: str = "" + usage: Optional[Dict[str, Any]] = None + duration_ms: Optional[int] = None + + +def run_claude( + *, + prompt: Optional[str], + model: str, + base_url: str, + api_key: str, + extra_env: Optional[Mapping[str, str]] = None, + extra_args: Optional[Sequence[str]] = None, + stdin_input: Optional[str] = None, + cli_path: str = CLAUDE_CLI_DEFAULT, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + runner: Optional[Any] = None, + rate_limiter: Optional[RateLimiter] = None, +) -> DriverResult: + """Invoke `claude` once in headless stream-JSON mode and return the result. + + The CLI is pointed at a LiteLLM proxy via `ANTHROPIC_BASE_URL` / + `ANTHROPIC_AUTH_TOKEN`, so the same code path exercises every provider + column — only the model id and the proxy's routing differ between + invocations. + + `runner` is an injection seam used by the unit tests: by default we call + `subprocess.run`, but the test suite swaps in a fake that yields canned + stream-JSON. Production callers should never set it. + + `rate_limiter` is the second injection seam: the cross-process + token-bucket limiter throttles outbound calls per provider so a + fully-parallel matrix run doesn't trip 429s. Defaults to the + process-wide singleton; unit tests pass a no-op limiter or one + backed by a tmp dir to keep tests hermetic. + """ + if prompt is None and stdin_input is None: + raise ValueError("must supply either `prompt` or `stdin_input`") + if prompt is not None and stdin_input is not None: + raise ValueError("must supply only one of `prompt` or `stdin_input`, not both") + if prompt is not None and not prompt: + raise ValueError("prompt must be a non-empty string when provided") + if stdin_input is not None and not stdin_input: + raise ValueError("stdin_input must be a non-empty string when provided") + if not model: + raise ValueError("model must be a non-empty string") + if not base_url: + raise ValueError("base_url must be a non-empty string") + if not api_key: + raise ValueError("api_key must be a non-empty string") + + # `claude --print` takes the prompt as the **last positional argument**. + # Flags must come before it, otherwise they're parsed as part of the + # prompt (or silently dropped, depending on the CLI version) and the + # tool_use / vision cells fail with confusing "no tool_use observed" + # errors. Build the flag list first, then append the prompt last. + # + # When `extra_args` contains a *variadic* flag like `--allowed-tools + # WebSearch` (commander.js's ``), the parser greedily + # consumes every subsequent token as part of the variadic list — so + # the prompt would be eaten as a tool name. Inserting `--` before + # the prompt terminates option parsing and leaves the prompt as a + # plain positional, which works for variadic and non-variadic flags + # alike. + cmd: List[str] = [ + cli_path, + "--print", + "--output-format", + "stream-json", + "--verbose", + "--model", + model, + ] + if extra_args: + cmd.extend(extra_args) + if prompt is not None: + cmd.append("--") + cmd.append(prompt) + + # Build a minimal env for the CLI subprocess: only the allowlisted + # process-runtime vars from os.environ, plus the explicit proxy + # creds, plus any caller-supplied overrides. See _CLI_ENV_ALLOWLIST + # above for the security rationale. + env: Dict[str, str] = { + key: os.environ[key] for key in _CLI_ENV_ALLOWLIST if key in os.environ + } + env["ANTHROPIC_BASE_URL"] = base_url + env["ANTHROPIC_AUTH_TOKEN"] = api_key + # Hand the CLI a fresh empty HOME so a compromised claude package + # or a model-directed Read tool call can't see the runtime user's + # real dotfiles. Created here, removed in the `finally` below + # regardless of how the subprocess exits. + isolated_home = _make_isolated_home() + env["HOME"] = isolated_home + if extra_env: + env.update(extra_env) + + # Throttle by provider *before* launching the CLI. Doing this here + # (rather than per-test) means every code path that lands on + # `run_claude` is rate-limited automatically — including + # `run_claude_models_parallel`, which is the hot path during the + # full matrix run. + limiter = rate_limiter if rate_limiter is not None else get_default_limiter() + provider = infer_provider(model) + limiter.acquire(provider) + + run_fn = runner or subprocess.run + try: + try: + completed = run_fn( + cmd, + env=env, + input=stdin_input, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except FileNotFoundError as exc: + raise ClaudeCLIError( + f"claude CLI not found at {cli_path!r}; install with `npm i -g @anthropic-ai/claude-code`" + ) from exc + except subprocess.TimeoutExpired as exc: + raise ClaudeCLIError(f"claude CLI timed out after {timeout}s") from exc + finally: + # Best-effort cleanup. If the subprocess wrote a `.claude/` + # session dir under the isolated HOME, we remove it here so + # parallel matrix runs don't accumulate per-call tmpdirs. + # `ignore_errors=True` because rmtree races with any + # not-yet-reaped child (SIGTERM'd `claude` on a host-side + # timeout) are benign — the next matrix run starts from a + # fresh tmpdir anyway. + shutil.rmtree(isolated_home, ignore_errors=True) + + events = _parse_stream_json(completed.stdout or "") + text = _extract_assistant_text(events) + usage = _extract_usage(events) + + return DriverResult( + text=text, + events=events, + exit_code=completed.returncode, + stderr=completed.stderr or "", + usage=usage, + ) + + +ModelResult = Union[DriverResult, ClaudeCLIError] + + +def run_claude_models_parallel( + *, + models: Sequence[str], + prompt: Optional[str], + base_url: str, + api_key: str, + extra_env: Optional[Mapping[str, str]] = None, + extra_args: Optional[Sequence[str]] = None, + stdin_input: Optional[str] = None, + cli_path: str = CLAUDE_CLI_DEFAULT, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + runner: Optional[Callable[..., Any]] = None, +) -> Dict[str, ModelResult]: + """Invoke `run_claude` for every `models[i]` concurrently and collect outcomes. + + Each `claude` CLI invocation is a long-lived subprocess that spends + almost all of its time waiting on the upstream API; running the + three Claude tiers in parallel cuts the per-cell wall time roughly + threefold without changing what each invocation does. + + Threads (rather than asyncio) are the right primitive here because + `subprocess.run` releases the GIL while it waits, and we want to + keep the synchronous CLI driver unchanged so unit tests can keep + injecting a fake `runner`. + + Returns a dict keyed by model id. Each value is either the + `DriverResult` produced by `run_claude` or the `ClaudeCLIError` + that aborted that model's run — callers decide how to map either + into a `compat_result` entry. The shared kwargs (prompt, env, args, + timeout, runner) are forwarded verbatim so the per-model wire is + identical to what the sequential path produces. + """ + if not models: + raise ValueError("models must be a non-empty sequence") + + def _one(model: str) -> Tuple[str, ModelResult, float]: + # Per-model wall clock: this is what the matrix run actually pays for. + # We record it whether the run succeeded or raised so the breakdown + # log below covers both code paths and surfaces "which model is the + # long pole?" without requiring per-test instrumentation. + started = time.monotonic() + try: + result = run_claude( + prompt=prompt, + model=model, + base_url=base_url, + api_key=api_key, + extra_env=extra_env, + extra_args=extra_args, + stdin_input=stdin_input, + cli_path=cli_path, + timeout=timeout, + runner=runner, + ) + elapsed = time.monotonic() - started + # Stamp the duration onto the DriverResult so callers (tests, + # diagnostics) can attribute slow cells without re-timing. + result.duration_ms = int(elapsed * 1000) + return model, result, elapsed + except ClaudeCLIError as exc: + elapsed = time.monotonic() - started + return model, exc, elapsed + except Exception as exc: + # Honor the documented "errors as values" contract for any + # exception type — not just ClaudeCLIError. The rate + # limiter does file I/O (OSError), `infer_provider` can + # raise ValueError on edge-case model strings, and a future + # bug elsewhere in the call stack must not abort the entire + # parallel batch and lose the other models' outcomes. + elapsed = time.monotonic() - started + wrapped = ClaudeCLIError( + f"unexpected error running model {model!r}: " + f"{type(exc).__name__}: {exc}" + ) + wrapped.__cause__ = exc + return model, wrapped, elapsed + + outcomes: Dict[str, ModelResult] = {} + durations: Dict[str, float] = {} + overall_started = time.monotonic() + with ThreadPoolExecutor(max_workers=len(models)) as pool: + futures = [pool.submit(_one, model) for model in models] + for future in as_completed(futures): + model, outcome, elapsed = future.result() + outcomes[model] = outcome + durations[model] = elapsed + overall_elapsed = time.monotonic() - overall_started + + _log_parallel_breakdown(models, durations, outcomes, overall_elapsed) + return outcomes + + +def _log_parallel_breakdown( + models: Sequence[str], + durations: Mapping[str, float], + outcomes: Mapping[str, ModelResult], + overall_elapsed: float, +) -> None: + """Emit a one-block timing breakdown to stderr. + + Pytest only shows captured output for failing tests by default, but + `-s` surfaces it for passing tests too — which is exactly when you + care about "did parallelization actually help?". The block reports: + + - per-model wall time and outcome (ok / cli-error / non-zero exit) + - the slowest model (the parallel run's wall-time floor) + - the sum of sequential model times (what the old serial path + would have paid) + - the overall parallel wall time and the speedup ratio + + If one model dominates, `slowest ≈ overall ≈ sequential / 1`, and + the speedup will be near 1× — exactly the diagnostic that explains + "why didn't this get faster?". + """ + sequential_total = sum(durations.values()) + slowest_model = max(durations, key=durations.get) if durations else None + slowest = durations[slowest_model] if slowest_model else 0.0 + speedup = sequential_total / overall_elapsed if overall_elapsed > 0 else 0.0 + + lines: List[str] = [] + lines.append("[parallel] per-model wall time:") + for model in models: + elapsed = durations.get(model, 0.0) + outcome = outcomes.get(model) + if isinstance(outcome, ClaudeCLIError): + status = "cli-error" + elif isinstance(outcome, DriverResult): + status = f"exit={outcome.exit_code}" + else: + status = "missing" + lines.append(f" {model:<40s} {elapsed:6.2f}s ({status})") + if slowest_model is not None: + lines.append( + f"[parallel] slowest={slowest_model} ({slowest:.2f}s); " + f"sequential_sum={sequential_total:.2f}s; " + f"parallel_wall={overall_elapsed:.2f}s; " + f"speedup={speedup:.2f}x" + ) + print("\n".join(lines), file=sys.stderr, flush=True) + + +def _parse_stream_json(stdout: str) -> List[Dict[str, Any]]: + """Parse newline-delimited JSON emitted by `claude --output-format stream-json`. + + Lines that don't parse as JSON are silently skipped — the CLI occasionally + emits debug output we don't care about, and a single malformed line should + not abort the whole run. Real failure modes surface via exit code. + """ + events: List[Dict[str, Any]] = [] + for line in stdout.splitlines(): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(obj, dict): + events.append(obj) + return events + + +def _extract_assistant_text(events: Sequence[Mapping[str, Any]]) -> str: + """Concatenate the text content of every `assistant` event in order. + + The non-streaming `--print` path emits a single `assistant` event whose + `message.content` is a list of content blocks. We walk the blocks and + join every `text` block — the CLI prints other block types (e.g. + `tool_use`) which we ignore for the basic-messaging case. + """ + chunks: List[str] = [] + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if isinstance(content, str): + chunks.append(content) + continue + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") == "text" and isinstance(block.get("text"), str): + chunks.append(block["text"]) + return "".join(chunks) + + +def failure_diagnostic(result: "DriverResult", *, max_len: int = 800) -> str: + """Build a human-readable error string from a non-zero `claude` CLI run. + + The CLI is annoying to debug because the most useful failure signal + rarely lands on stderr. When the proxy returns an HTTP error, the CLI + swallows it into an `assistant`/`result` event on **stdout** with + `is_error: true` and a JSON-shaped `text` block — and exits non-zero. + Tests that only print `stderr.strip()` see an empty string, which is + exactly the situation that masked a misconfigured proxy in early + bring-up of the compat matrix. + + This helper concatenates the most useful diagnostic we can find, in + priority order: + + 1. `result.text` (the assistant's user-visible reply, which is where + API errors land in stream-json mode), trimmed + 2. `api_error_status` from any `result` event, if present + 3. `result.stderr`, trimmed + 4. `` as a last resort + + The output is truncated to `max_len` characters so a giant HTML 502 + page from a misbehaving load balancer doesn't blow up the matrix + JSON. + """ + pieces: List[str] = [f"exit={result.exit_code}"] + + # api_error_status only appears on the final `result` event when the + # CLI received an HTTP error from the upstream API. Surfacing it + # explicitly makes "is this a proxy/auth problem or a CLI problem?" + # answerable without re-reading the events list. + api_status = _extract_api_error_status(result.events) + if api_status is not None: + pieces.append(f"api_status={api_status}") + + text = (result.text or "").strip() + if text: + pieces.append(f"text={_truncate(text, max_len)}") + + stderr = (result.stderr or "").strip() + if stderr: + pieces.append(f"stderr={_truncate(stderr, max_len)}") + + if len(pieces) == 1: + pieces.append("(no diagnostic output)") + + return "; ".join(pieces) + + +def _extract_api_error_status( + events: Sequence[Mapping[str, Any]], +) -> Optional[int]: + """Return the `api_error_status` from the last `result` event, if any.""" + for event in reversed(list(events)): + if event.get("type") != "result": + continue + status = event.get("api_error_status") + if isinstance(status, int): + return status + return None + + +def _truncate(s: str, max_len: int) -> str: + if len(s) <= max_len: + return s + return s[:max_len] + "...(truncated)" + + +def _extract_usage(events: Sequence[Mapping[str, Any]]) -> Optional[Dict[str, Any]]: + """Return the most recent `usage` block seen on any event, if any. + + The CLI surfaces token + cache usage on the final `result` event for + non-streaming runs, but earlier events also carry partial usage in some + versions; taking the last non-empty one is the safe default. + """ + last: Optional[Dict[str, Any]] = None + for event in events: + usage = event.get("usage") + if isinstance(usage, dict) and usage: + last = usage + continue + message = event.get("message") + if isinstance(message, dict): + inner = message.get("usage") + if isinstance(inner, dict) and inner: + last = inner + return last diff --git a/tests/e2e/claude_code/conftest.py b/tests/e2e/claude_code/conftest.py new file mode 100644 index 00000000000..d2bfa1a54bf --- /dev/null +++ b/tests/e2e/claude_code/conftest.py @@ -0,0 +1,550 @@ +"""Pytest plumbing for the Claude Code compatibility matrix. + +Three responsibilities live here: + +1. The `compat_result` fixture — the only API a test author needs to learn. + Tests call `compat_result.set({"status": "pass"})` (or fail / not_applicable) + to report their outcome as a tagged union. Multi-model tests call + `.add(...)` once per Claude tier so each tier lands as its own row in + the results artifact. + +2. The `pytest_runtest_makereport` hook — captures each test's reported result, + infers (feature, provider) from the file path, and accumulates rows into + a per-process collector. At session end we serialize them to + `compat-results.json` (or a per-worker file under xdist) so the Matrix + JSON Builder can consume them. + +3. xdist coordination — when `pytest -n auto` is used, every worker writes + its own results shard and the controller merges them into the canonical + `compat-results.json` in `pytest_sessionfinish`. Without this, the + workers race on the same path and the artifact only reflects whichever + worker finished last. The same merge step also emits a rate-limit + summary that the binary-search helper consumes to decide whether the + current X/Y/Z values were too aggressive. + +The (feature, provider) inference comes from the test file path: the parent +directory name is the feature_id (matching `manifest.yaml`), and the file +stem after the leading `test_` is the provider id. This avoids per-file +metadata that drifts. +""" + +from __future__ import annotations + +import functools +import json +import os +import re +import sys +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, FrozenSet, List, Optional, Tuple + +import pytest +import yaml + +VALID_STATUSES = {"pass", "fail", "not_applicable", "not_tested"} +RESULTS_ARTIFACT_ENV = "COMPAT_RESULTS_PATH" +DEFAULT_ARTIFACT_PATH = "compat-results.json" +RATE_LIMIT_SUMMARY_ENV = "COMPAT_RATE_LIMIT_SUMMARY_PATH" +DEFAULT_RATE_LIMIT_SUMMARY_PATH = "compat-rate-limit-summary.json" + +# Heuristic: detect 429s and rate-limit-shaped errors anywhere in the +# error string. The CLI buries upstream errors in `assistant.message.content` +# text on stdout (see `failure_diagnostic`), so we don't get a structured +# status code in every code path — a regex over the joined error text is +# the most reliable signal we have. +# +# We also treat a CLI timeout (`claude CLI timed out after Ns`) as a +# rate-limit-shaped failure for binary-search purposes: in practice the +# only reason every model in a cell stalls past the timeout is the +# upstream collapsing under concurrency, which is exactly the situation +# the rate limiter is supposed to back off from. False positives on a +# genuinely slow upstream are tolerable here because the worst case is +# the binary search runs at a slightly lower rate than necessary. +_RATE_LIMIT_RE = re.compile( + r"(?:\b429\b|rate[\s_-]?limit|too\s+many\s+requests|throttl(?:ed|ing)|" + r"claude\s+CLI\s+timed\s+out)", + re.IGNORECASE, +) + + +@dataclass +class CompatResult: + """Per-test recorder for compatibility outcomes. + + Tests interact via `.set(...)` (single result) or `.add(...)` (one + result per Claude tier when the test fans the three models out in + parallel). `.value` and `.values` are read by the + `pytest_runtest_makereport` hook after the test body finishes. + + Multi-result usage exists because every cell in the compat matrix is + backed by three model invocations (Haiku/Sonnet/Opus) per (feature, + provider). When a test runs them concurrently in a single pytest + node, each model needs its own entry in the results artifact so the + matrix builder's per-cell aggregator can apply its "all three must + pass" rule. + """ + + value: Optional[Dict[str, Any]] = None + values: List[Dict[str, Any]] = field(default_factory=list) + + def set(self, result: Dict[str, Any]) -> None: + validated = self._validate(result) + self.value = validated + + def add(self, result: Dict[str, Any]) -> None: + """Append one model's outcome to the per-test results list. + + Use this when a single test exercises multiple Claude tiers + concurrently and needs to report one outcome per tier. The + conftest hook will emit one entry per appended result. + """ + validated = self._validate(result) + self.values.append(validated) + + @staticmethod + def _validate(result: Dict[str, Any]) -> Dict[str, Any]: + if not isinstance(result, dict): + raise TypeError("compat_result requires a dict") + status = result.get("status") + if status not in VALID_STATUSES: + raise ValueError( + f"compat_result status must be one of {sorted(VALID_STATUSES)}, " + f"got {status!r}" + ) + if status == "fail" and not result.get("error"): + raise ValueError("compat_result {'status': 'fail'} requires 'error'") + if status == "not_applicable" and not result.get("reason"): + raise ValueError( + "compat_result {'status': 'not_applicable'} requires 'reason'" + ) + return dict(result) + + def collected(self) -> List[Dict[str, Any]]: + """Return every result reported during the test, preserving order. + + Multi-model tests use `.add(...)` per model; legacy tests use + `.set(...)` once. We surface both shapes in a single list so + the makereport hook only has to think about a list of results. + """ + if self.values: + return list(self.values) + if self.value is not None: + return [dict(self.value)] + return [] + + +@dataclass +class _CollectedResult: + feature_id: str + provider: str + nodeid: str + result: Dict[str, Any] + + +@dataclass +class _Collector: + items: List[_CollectedResult] = field(default_factory=list) + + +_COLLECTOR = _Collector() + + +@pytest.fixture +def compat_result() -> CompatResult: + """Per-test recorder for the (feature, provider) outcome. + + Tests should call `compat_result.set({"status": "pass"})` (or fail / + not_applicable) before returning. If a test exits without calling `.set()` + the harness records `status="fail"` with an explanatory error so that + every collected node maps to a real cell. + """ + return CompatResult() + + +@functools.lru_cache(maxsize=1) +def _manifest_feature_ids() -> FrozenSet[str]: + """Return the set of feature_ids declared in `manifest.yaml`. + + Used as a positive filter so only directories that correspond to a + real matrix row contribute results — utility/support directories + (e.g. `cron_vm`, `_driver_unit_tests`) are dropped regardless of + naming convention, and the rate-limit summary stays clean. + + Returns an empty set if the manifest is missing or malformed; the + caller treats that as "no path is a feature path", which is the + safe default — we'd rather drop a real result than pollute the + artifact with a garbage cell. + """ + manifest_path = Path(__file__).resolve().parent / "manifest.yaml" + try: + raw = yaml.safe_load(manifest_path.read_text()) + except (OSError, yaml.YAMLError): + return frozenset() + if not isinstance(raw, dict): + return frozenset() + features = raw.get("features") + if not isinstance(features, list): + return frozenset() + return frozenset( + entry["id"] + for entry in features + if isinstance(entry, dict) and isinstance(entry.get("id"), str) + ) + + +def _infer_feature_and_provider(node_path: Path) -> Optional[tuple]: + """Infer (feature_id, provider) from a test file path. + + Path shape: tests/e2e/claude_code//test_.py + Returns None if the file is not a per-feature test (e.g. unit tests + under `_driver_unit_tests/` or support code under `cron_vm/`), so + those don't pollute the matrix artifact. We positively filter the + parent directory against `manifest.yaml` rather than relying on + naming conventions, because non-feature siblings don't all share + an underscore prefix. + """ + name = node_path.name + if not name.startswith("test_") or not name.endswith(".py"): + return None + provider = name[len("test_") : -len(".py")] + feature_id = node_path.parent.name + if feature_id not in _manifest_feature_ids(): + return None + return feature_id, provider + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + """Capture compat_result reports at end-of-test and remember them for the artifact. + + A single test may report multiple results (one per Claude tier when + the three are run in parallel inside one node). We emit one + `_CollectedResult` per reported entry so the matrix builder's + per-cell aggregator sees the same shape it would have seen if the + test were parametrized — every model lands in the artifact. + """ + outcome = yield + report = outcome.get_result() + # We record on two phases: + # - "call": the normal end-of-test path. + # - "setup" but only on failure: fixture/import errors that prevent the + # test body from running. Without recording these, a broken setup + # silently becomes "not_tested" in the published matrix instead of + # "fail". Teardown is ignored — by then "call" already recorded the + # outcome, and a teardown-only failure (e.g. fixture finalizer) is + # not a cell-level signal. + if report.when == "setup": + if not report.failed: + return + elif report.when != "call": + return + + # A skipped test (e.g. `pytest.skip(...)` called inside the body or + # by a `pytest.mark.skipif` evaluated at call time) is neither a + # pass nor a fail — it just didn't run. Recording it as anything + # here would produce a spurious row (the not-failed/empty-collected + # branch below would mark it as a fail with "test passed without + # reporting via compat_result"), so bail out and let the cell stay + # "not_tested" in the published matrix. + if report.skipped: + return + + inferred = _infer_feature_and_provider(Path(str(item.path))) + if inferred is None: + return + feature_id, provider = inferred + + fixture = item.funcargs.get("compat_result") if hasattr(item, "funcargs") else None + collected: List[Dict[str, Any]] = ( + fixture.collected() if isinstance(fixture, CompatResult) else [] + ) + + if report.failed and not any(entry.get("status") == "fail" for entry in collected): + # The test body (or setup) raised and the test author hasn't + # already recorded a fail row via `.add(...)`. If the test had + # recorded only per-model passes before crashing, those partial + # entries would aggregate to "pass" and hide the crash from the + # published matrix; append an explicit "fail" row so the cell + # aggregator (which gives precedence to any fail) surfaces the + # breakage. We skip the append when a fail row is already + # present so that the common pattern — `.add({"status": "fail", + # ...})` per failing model, then `pytest.fail("; ".join(...))` + # to surface them — doesn't produce a phantom duplicate row. + collected = collected + [ + { + "status": "fail", + "error": (str(report.longrepr) if report.longrepr else "test failed"), + } + ] + elif not report.failed and not collected: + collected = [ + { + "status": "fail", + "error": "test passed without reporting via compat_result; " + "every compat test must report a status.", + } + ] + + for reported in collected: + _COLLECTOR.items.append( + _CollectedResult( + feature_id=feature_id, + provider=provider, + nodeid=report.nodeid, + result=reported, + ) + ) + + +def _is_xdist_worker(session) -> bool: + """Return True iff the current pytest session is an xdist worker. + + The standard idiom is to look up `workerinput` on the config; the + controller process doesn't have it, the workers do. We deliberately + don't `import xdist` because the suite must keep running when xdist + isn't installed at all. + """ + return hasattr(session.config, "workerinput") + + +def _xdist_worker_id(session) -> Optional[str]: + info = getattr(session.config, "workerinput", None) + if not info: + return None + return info.get("workerid") + + +def _shard_dir(artifact_path: Path) -> Path: + """Workers write their shards next to the canonical results path. + + Putting shards in a sibling directory (rather than inline JSON + files in the same dir) keeps the controller's merge step simple + — it just lists `*.json` in `.shards/` — and avoids + accidental shard/canonical filename collisions. + """ + return artifact_path.with_name(artifact_path.name + ".shards") + + +def _serialize_items(items: List["_CollectedResult"]) -> List[Dict[str, Any]]: + return [ + { + "feature_id": item.feature_id, + "provider": item.provider, + "nodeid": item.nodeid, + "result": item.result, + } + for item in items + ] + + +def _build_rate_limit_summary( + rows: List[Dict[str, Any]], +) -> Dict[str, Any]: + """Aggregate per-provider rate-limit signals from the result rows. + + We classify any failure whose error string matches `_RATE_LIMIT_RE` + as a "rate-limited" failure. The binary-search helper reads this + summary to decide whether the current X/Y/Z values were too + aggressive: if any provider has `rate_limited > 0`, the harness + should back off that provider's rate and retry. + + Returns a dict shaped: + + { + "totals": {"pass": N, "fail": N, "rate_limited": N, ...}, + "per_provider": { + "anthropic": {"pass": ..., "fail": ..., "rate_limited": ...}, + ... + }, + "rate_limited_examples": [ + {"feature_id": ..., "provider": ..., "error": "..."}, ... + ], + } + """ + totals: Counter = Counter() + per_provider: Dict[str, Counter] = defaultdict(Counter) + rate_limited_examples: List[Dict[str, Any]] = [] + + for row in rows: + result = row.get("result") or {} + status = result.get("status") or "unknown" + provider = row.get("provider") or "unknown" + totals[status] += 1 + per_provider[provider][status] += 1 + + if status == "fail": + error = str(result.get("error") or "") + if _RATE_LIMIT_RE.search(error): + totals["rate_limited"] += 1 + per_provider[provider]["rate_limited"] += 1 + # Cap examples so a stuck-throttled run doesn't write + # a multi-MB summary file the helper has to slurp. + if len(rate_limited_examples) < 25: + rate_limited_examples.append( + { + "feature_id": row.get("feature_id"), + "provider": provider, + "error": error[:500], + } + ) + + return { + "totals": dict(totals), + "per_provider": {p: dict(c) for p, c in per_provider.items()}, + "rate_limited_examples": rate_limited_examples, + } + + +def _print_rate_limit_summary(summary: Dict[str, Any]) -> None: + """Emit a human-readable per-provider table to stderr. + + Pytest only captures stderr when `-s` isn't set; we deliberately + write here anyway because the binary-search workflow runs pytest + with `-q` and grep-checks the structured JSON artifact, while a + human running locally with `-s` sees the same numbers inline. + """ + totals = summary.get("totals", {}) + per_provider = summary.get("per_provider", {}) + lines: List[str] = [] + lines.append("[compat] session totals:") + for status in ("pass", "fail", "rate_limited", "not_applicable", "not_tested"): + if status in totals: + lines.append(f" {status:<16s} {totals[status]}") + if per_provider: + lines.append("[compat] per-provider breakdown:") + for provider in sorted(per_provider): + counts = per_provider[provider] + parts = " ".join( + f"{k}={v}" + for k, v in sorted(counts.items()) + if k != "not_tested" or v > 0 + ) + lines.append(f" {provider:<20s} {parts}") + if totals.get("rate_limited", 0): + lines.append( + "[compat] WARNING: at least one cell hit a rate-limit-shaped error; " + "lower the corresponding LITELLM_COMPAT_RATE_ and retry" + ) + print("\n".join(lines), file=sys.stderr, flush=True) + + +def pytest_sessionstart(session): + """Reset per-session state before tests run. + + Two responsibilities: + + 1. Clear the module-level `_COLLECTOR` singleton, which survives + across `pytest.main()` invocations within the same Python + process. Without this reset, results from a prior session + would leak into the next run's `compat-results.json` artifact. + + 2. Remove stale per-worker shards from any prior session. Without + this, a previous run's shard directory leaks into the next + `pytest_sessionfinish` merge — yielding a `compat-results.json` + that includes results from runs that aren't part of the current + session, and a misleading rate-limit summary that re-flags + failures the user already saw and addressed. Only the + controller (non-xdist-worker) clears; workers must not race + the controller while it's wiping the directory. + """ + _COLLECTOR.items.clear() + _manifest_feature_ids.cache_clear() + + if _is_xdist_worker(session): + return + artifact_path = Path(os.environ.get(RESULTS_ARTIFACT_ENV) or DEFAULT_ARTIFACT_PATH) + shard_dir = _shard_dir(artifact_path) + if not shard_dir.exists(): + return + for stale in shard_dir.glob("*.json"): + try: + stale.unlink() + except OSError: + # If we can't remove a stale shard (permissions, race with + # an unrelated process), keep going — the merge step is + # robust to malformed shards, and a stale row landing in + # the artifact is recoverable; aborting the session isn't. + continue + + +def pytest_sessionfinish(session, exitstatus): + """Write the per-process results shard, then merge if we're the controller. + + Worker processes (xdist `gw0`, `gw1`, ...) only write their shard + under `.shards/.json`. The controller writes + its own shard if it ran any tests itself, then walks the shards + directory and produces the canonical `compat-results.json` plus + the rate-limit summary. Single-process runs (no xdist) take the + same code path with a single shard, so behavior is consistent. + + Skip when no compat results were collected — this conftest is + loaded for every test under `tests/e2e/claude_code/`, including sibling + unit-test trees (e.g. `_driver_unit_tests/`). Writing an empty + artifact would silently overwrite a real artifact from a prior + compat-test run on the same checkout. + + The xdist controller hits this hook with `_COLLECTOR.items` empty + (it never executes tests itself) and `_is_xdist_worker` False, so + we additionally allow the merge step to run when worker shards + are already on disk — otherwise the canonical artifact would + never be produced under `pytest -n auto`. + """ + artifact_path = Path(os.environ.get(RESULTS_ARTIFACT_ENV) or DEFAULT_ARTIFACT_PATH) + shard_dir = _shard_dir(artifact_path) + has_worker_shards = shard_dir.is_dir() and any(shard_dir.glob("*.json")) + if not _COLLECTOR.items and not _is_xdist_worker(session) and not has_worker_shards: + return + shard_dir.mkdir(parents=True, exist_ok=True) + + worker_id = _xdist_worker_id(session) or "main" + shard_path = shard_dir / f"{worker_id}.json" + shard_path.write_text( + json.dumps( + { + "schema_version": "1", + "worker_id": worker_id, + "results": _serialize_items(_COLLECTOR.items), + }, + indent=2, + sort_keys=True, + ) + ) + + # Workers stop here. The controller merges; if we're not running + # under xdist, we are effectively the controller. + if _is_xdist_worker(session): + return + + merged_rows: List[Dict[str, Any]] = [] + for shard_file in sorted(shard_dir.glob("*.json")): + try: + shard = json.loads(shard_file.read_text()) + except (OSError, ValueError): + continue + rows = shard.get("results") + if isinstance(rows, list): + merged_rows.extend(rows) + + # Skip writing artifact + summary entirely for unit-test-only runs + # (no per-feature compat rows). Otherwise every `pytest tests/...` + # run — including local unit-test invocations — would silently + # overwrite a real artifact from a prior compat-test run. + if not merged_rows: + return + + artifact_path.write_text( + json.dumps( + {"schema_version": "1", "results": merged_rows}, + indent=2, + sort_keys=True, + ) + ) + + summary = _build_rate_limit_summary(merged_rows) + summary_path = Path( + os.environ.get(RATE_LIMIT_SUMMARY_ENV) or DEFAULT_RATE_LIMIT_SUMMARY_PATH + ) + summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True)) + _print_rate_limit_summary(summary) diff --git a/tests/e2e/claude_code/count_tokens/__init__.py b/tests/e2e/claude_code/count_tokens/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/count_tokens/test_anthropic.py b/tests/e2e/claude_code/count_tokens/test_anthropic.py new file mode 100644 index 00000000000..3508063459c --- /dev/null +++ b/tests/e2e/claude_code/count_tokens/test_anthropic.py @@ -0,0 +1,94 @@ +"""count_tokens x Anthropic. + +HTTP-probe row. Unlike the CLI-driven rows, this test never invokes +the `claude` CLI: it `POST`s directly to +`{proxy}/v1/messages/count_tokens` for each Claude tier and asserts +the response is shaped `{"input_tokens": }`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/count_tokens/test_anthropic.py + ^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider + +Why HTTP probe instead of CLI: + +Claude Code calls `count_tokens` internally to compute budget / +context-window usage display, but the result is consumed by the CLI +in-process and never appears in stream-json events. There is no CLI +flag that emits the count to stdout in a way our existing +stream-json parser can pick up, so we can't test the endpoint round +trip through the CLI surface. + +The proxy *is* expected to expose `/v1/messages/count_tokens` for +every Claude-style provider it routes to -- LiteLLM has historically +had provider-specific bugs in this endpoint (Vertex AI `count_tokens` +returned 400 to proxy gateways; see Claude Code release notes 2.1.121). +Treating it as a matrix row keeps regressions in the cron's daily +diff. + +The cell goes red if *any* tier's probe fails the minimal shape +check; the matrix's per-cell aggregator handles that automatically. +Three tiers run sequentially because count_tokens is cheap (<100ms +per request typical) and the parallelization that matters for the +CLI rows isn't useful here. +""" + +from __future__ import annotations + +import os + +import pytest + +from claude_code.http_probe import ( + assert_count_tokens_shape, + probe_count_tokens, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + + +def test_count_tokens_anthropic(compat_result): + """Probe `/v1/messages/count_tokens` for each Anthropic tier and + assert the response shape.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + failures = [] + for model in ANTHROPIC_MODELS: + result = probe_count_tokens( + base_url=base_url, api_key=api_key, model=model + ) + shape_error = assert_count_tokens_shape(result) + if shape_error is not None: + error = f"[{model}] count_tokens probe failed: {shape_error}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/count_tokens/test_azure.py b/tests/e2e/claude_code/count_tokens/test_azure.py new file mode 100644 index 00000000000..2b8707b50b0 --- /dev/null +++ b/tests/e2e/claude_code/count_tokens/test_azure.py @@ -0,0 +1,94 @@ +"""count_tokens x Azure (Microsoft Foundry). + +HTTP-probe row. Unlike the CLI-driven rows, this test never invokes +the `claude` CLI: it `POST`s directly to +`{proxy}/v1/messages/count_tokens` for each Claude tier and asserts +the response is shaped `{"input_tokens": }`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/count_tokens/test_azure.py + ^^^^^^^^^^^^ ^^^^^ + feature_id provider + +Why HTTP probe instead of CLI: + +Claude Code calls `count_tokens` internally to compute budget / +context-window usage display, but the result is consumed by the CLI +in-process and never appears in stream-json events. There is no CLI +flag that emits the count to stdout in a way our existing +stream-json parser can pick up, so we can't test the endpoint round +trip through the CLI surface. + +The proxy *is* expected to expose `/v1/messages/count_tokens` for +every Claude-style provider it routes to -- LiteLLM has historically +had provider-specific bugs in this endpoint (Vertex AI `count_tokens` +returned 400 to proxy gateways; see Claude Code release notes 2.1.121). +Treating it as a matrix row keeps regressions in the cron's daily +diff. + +The cell goes red if *any* tier's probe fails the minimal shape +check; the matrix's per-cell aggregator handles that automatically. +Three tiers run sequentially because count_tokens is cheap (<100ms +per request typical) and the parallelization that matters for the +CLI rows isn't useful here. +""" + +from __future__ import annotations + +import os + +import pytest + +from claude_code.http_probe import ( + assert_count_tokens_shape, + probe_count_tokens, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + + +def test_count_tokens_azure(compat_result): + """Probe `/v1/messages/count_tokens` for each Azure (Microsoft Foundry) tier and + assert the response shape.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + failures = [] + for model in AZURE_MODELS: + result = probe_count_tokens( + base_url=base_url, api_key=api_key, model=model + ) + shape_error = assert_count_tokens_shape(result) + if shape_error is not None: + error = f"[{model}] count_tokens probe failed: {shape_error}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py b/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py new file mode 100644 index 00000000000..4221773ead2 --- /dev/null +++ b/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py @@ -0,0 +1,94 @@ +"""count_tokens x Bedrock (Converse). + +HTTP-probe row. Unlike the CLI-driven rows, this test never invokes +the `claude` CLI: it `POST`s directly to +`{proxy}/v1/messages/count_tokens` for each Claude tier and asserts +the response is shaped `{"input_tokens": }`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/count_tokens/test_bedrock_converse.py + ^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider + +Why HTTP probe instead of CLI: + +Claude Code calls `count_tokens` internally to compute budget / +context-window usage display, but the result is consumed by the CLI +in-process and never appears in stream-json events. There is no CLI +flag that emits the count to stdout in a way our existing +stream-json parser can pick up, so we can't test the endpoint round +trip through the CLI surface. + +The proxy *is* expected to expose `/v1/messages/count_tokens` for +every Claude-style provider it routes to -- LiteLLM has historically +had provider-specific bugs in this endpoint (Vertex AI `count_tokens` +returned 400 to proxy gateways; see Claude Code release notes 2.1.121). +Treating it as a matrix row keeps regressions in the cron's daily +diff. + +The cell goes red if *any* tier's probe fails the minimal shape +check; the matrix's per-cell aggregator handles that automatically. +Three tiers run sequentially because count_tokens is cheap (<100ms +per request typical) and the parallelization that matters for the +CLI rows isn't useful here. +""" + +from __future__ import annotations + +import os + +import pytest + +from claude_code.http_probe import ( + assert_count_tokens_shape, + probe_count_tokens, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + + +def test_count_tokens_bedrock_converse(compat_result): + """Probe `/v1/messages/count_tokens` for each Bedrock (Converse) tier and + assert the response shape.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + result = probe_count_tokens( + base_url=base_url, api_key=api_key, model=model + ) + shape_error = assert_count_tokens_shape(result) + if shape_error is not None: + error = f"[{model}] count_tokens probe failed: {shape_error}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py b/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py new file mode 100644 index 00000000000..cc70bf12392 --- /dev/null +++ b/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py @@ -0,0 +1,94 @@ +"""count_tokens x Bedrock (Invoke). + +HTTP-probe row. Unlike the CLI-driven rows, this test never invokes +the `claude` CLI: it `POST`s directly to +`{proxy}/v1/messages/count_tokens` for each Claude tier and asserts +the response is shaped `{"input_tokens": }`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py + ^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider + +Why HTTP probe instead of CLI: + +Claude Code calls `count_tokens` internally to compute budget / +context-window usage display, but the result is consumed by the CLI +in-process and never appears in stream-json events. There is no CLI +flag that emits the count to stdout in a way our existing +stream-json parser can pick up, so we can't test the endpoint round +trip through the CLI surface. + +The proxy *is* expected to expose `/v1/messages/count_tokens` for +every Claude-style provider it routes to -- LiteLLM has historically +had provider-specific bugs in this endpoint (Vertex AI `count_tokens` +returned 400 to proxy gateways; see Claude Code release notes 2.1.121). +Treating it as a matrix row keeps regressions in the cron's daily +diff. + +The cell goes red if *any* tier's probe fails the minimal shape +check; the matrix's per-cell aggregator handles that automatically. +Three tiers run sequentially because count_tokens is cheap (<100ms +per request typical) and the parallelization that matters for the +CLI rows isn't useful here. +""" + +from __future__ import annotations + +import os + +import pytest + +from claude_code.http_probe import ( + assert_count_tokens_shape, + probe_count_tokens, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + + +def test_count_tokens_bedrock_invoke(compat_result): + """Probe `/v1/messages/count_tokens` for each Bedrock (Invoke) tier and + assert the response shape.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + result = probe_count_tokens( + base_url=base_url, api_key=api_key, model=model + ) + shape_error = assert_count_tokens_shape(result) + if shape_error is not None: + error = f"[{model}] count_tokens probe failed: {shape_error}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/count_tokens/test_vertex_ai.py b/tests/e2e/claude_code/count_tokens/test_vertex_ai.py new file mode 100644 index 00000000000..8c2678f7010 --- /dev/null +++ b/tests/e2e/claude_code/count_tokens/test_vertex_ai.py @@ -0,0 +1,94 @@ +"""count_tokens x Vertex AI. + +HTTP-probe row. Unlike the CLI-driven rows, this test never invokes +the `claude` CLI: it `POST`s directly to +`{proxy}/v1/messages/count_tokens` for each Claude tier and asserts +the response is shaped `{"input_tokens": }`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/count_tokens/test_vertex_ai.py + ^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider + +Why HTTP probe instead of CLI: + +Claude Code calls `count_tokens` internally to compute budget / +context-window usage display, but the result is consumed by the CLI +in-process and never appears in stream-json events. There is no CLI +flag that emits the count to stdout in a way our existing +stream-json parser can pick up, so we can't test the endpoint round +trip through the CLI surface. + +The proxy *is* expected to expose `/v1/messages/count_tokens` for +every Claude-style provider it routes to -- LiteLLM has historically +had provider-specific bugs in this endpoint (Vertex AI `count_tokens` +returned 400 to proxy gateways; see Claude Code release notes 2.1.121). +Treating it as a matrix row keeps regressions in the cron's daily +diff. + +The cell goes red if *any* tier's probe fails the minimal shape +check; the matrix's per-cell aggregator handles that automatically. +Three tiers run sequentially because count_tokens is cheap (<100ms +per request typical) and the parallelization that matters for the +CLI rows isn't useful here. +""" + +from __future__ import annotations + +import os + +import pytest + +from claude_code.http_probe import ( + assert_count_tokens_shape, + probe_count_tokens, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + + +def test_count_tokens_vertex_ai(compat_result): + """Probe `/v1/messages/count_tokens` for each Vertex AI tier and + assert the response shape.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + failures = [] + for model in VERTEX_AI_MODELS: + result = probe_count_tokens( + base_url=base_url, api_key=api_key, model=model + ) + shape_error = assert_count_tokens_shape(result) + if shape_error is not None: + error = f"[{model}] count_tokens probe failed: {shape_error}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/cron_vm/build_matrix.py b/tests/e2e/claude_code/cron_vm/build_matrix.py new file mode 100644 index 00000000000..128f041cced --- /dev/null +++ b/tests/e2e/claude_code/cron_vm/build_matrix.py @@ -0,0 +1,50 @@ +"""Tiny CLI wrapper around `claude_code.matrix_builder.build_from_paths`. + +Exists only so `run_daily.sh` can hand the version metadata + paths into +the matrix builder without re-implementing it in bash. All real logic +lives in `matrix_builder.py`, which has its own unit tests under +`_builder_unit_tests/`. + +Invoked from the cron worktree (where `uv sync` has installed pyyaml), +not the dev checkout — the bash script `cd`s into the worktree before +`uv run python`-ing this file. +""" + +from __future__ import annotations + +import argparse +import datetime +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from claude_code.matrix_builder import build_from_paths # noqa: E402 # import needs the sys.path bootstrap above + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--results", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--litellm-version", required=True) + parser.add_argument("--claude-code-version", required=True) + args = parser.parse_args() + + generated_at = datetime.datetime.now(datetime.timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + build_from_paths( + manifest_path=args.manifest, + results_path=args.results, + litellm_version=args.litellm_version, + claude_code_version=args.claude_code_version, + generated_at=generated_at, + output_path=args.output, + ) + print(f"wrote {args.output}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example new file mode 100644 index 00000000000..11633810533 --- /dev/null +++ b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example @@ -0,0 +1,50 @@ +# Environment file consumed by `litellm-compat-matrix.service`. +# +# Install at `/etc/litellm-compat-matrix.env` and chmod 0600. +# `EnvironmentFile=-` in the unit means the service is allowed to start +# even if this file is missing, but the populator will fail at the +# first provider request without these credentials. + +# Anthropic +ANTHROPIC_API_KEY= + +# Bedrock (invoke + converse columns). +# Use Anthropic's Bedrock API-key passthrough (long-lived bearer token). +# No AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY required for the matrix -- +# both the LiteLLM invoke and converse routes pick up +# AWS_BEARER_TOKEN_BEDROCK when present. +AWS_BEARER_TOKEN_BEDROCK= +AWS_REGION_NAME=us-east-1 + +# Vertex AI. +# On the GCP VM, the default service-account ADC from the metadata server +# is used -- no JSON key file is needed. If you ever need to run outside +# GCP, also export GOOGLE_APPLICATION_CREDENTIALS=/path/to/sa.json. +VERTEXAI_PROJECT= +VERTEXAI_LOCATION=global + +# Microsoft Foundry (Azure column) +AZURE_FOUNDRY_API_KEY= +AZURE_FOUNDRY_API_BASE= + +# REQUIRED for publishing: PAT for the `agent-shin` user, used to push +# the daily compat-matrix branch to its fork (agent-shin/litellm-docs) +# and open the cross-repo PR against BerriAI/litellm-docs. Scopes: +# classic `repo` + `workflow`, or fine-grained on agent-shin/litellm-docs +# with Contents:RW + Pull requests:RW + Workflows:RW. +# Skip by setting SKIP_PUBLISH=1 (publishes nothing; only writes the +# matrix JSON locally). +AGENT_SHIN_GITHUB_TOKEN= + +# Optional: lifts the unauthenticated rate limit on the GitHub Releases +# API used by `resolver.py`. Any token works (read-only). Not required. +# GITHUB_TOKEN= + +# Optional overrides; defaults are sensible for the cron VM. +# PROXY_PORT=4100 +# LITELLM_WORKTREE=/home/mateo/litellm-cron-worktree +# DOCS_REPO=BerriAI/litellm-docs +# DOCS_BRANCH=main +# DOCS_TARGET_PATH=src/data/compatibility-matrix.json +# FORK_OWNER=agent-shin +# FORK_REPO=agent-shin/litellm-docs diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service new file mode 100644 index 00000000000..c05ece90f50 --- /dev/null +++ b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service @@ -0,0 +1,141 @@ +# systemd service for the Claude Code compatibility-matrix populator. +# +# Triggered by `litellm-compat-matrix.timer`; not started directly. The +# unit is a `Type=oneshot` so the timer's `OnCalendar=` semantics +# describe "run once per day" cleanly — there's no long-lived daemon to +# supervise; each invocation runs the populator end-to-end and exits. +# +# Install +# ------- +# +# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/ +# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/ +# sudo systemctl daemon-reload +# sudo systemctl enable --now litellm-compat-matrix.timer +# +# Paths are hard-coded to /home/mateo rather than using systemd's %h +# specifier. Why: in *system* units (this one), %h is expanded at +# parse time against the *manager's* home -- which is /root for PID 1 +# -- and *not* against the User= directive. That mismatch makes +# ReadWritePaths point at /root/.cache (which doesn't exist), causing +# the namespace setup to fail with status=226/NAMESPACE before the +# script ever runs. The runtime user (`User=mateo`) must: +# +# * have a checkout of `BerriAI/litellm` at `~/litellm/litellm` so the +# publisher module is importable; +# * have a uv venv at `~/litellm/litellm/.venv` (created by +# `uv sync --frozen` inside that checkout once); +# * have `gh` already authenticated against an account with +# `pull-requests: write` on `BerriAI/litellm-docs`; +# * have provider credentials exported in `/etc/litellm-compat-matrix.env` +# (see `litellm-compat-matrix.env.example` in this directory). + +[Unit] +Description=Claude Code compatibility-matrix populator (oneshot) +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +User=mateo +Group=mateo + +# Provider credentials + any gh/PROXY_PORT overrides live here. Format +# is the standard `KEY=value` one line per env var. +EnvironmentFile=-/etc/litellm-compat-matrix.env + +# systemd starts with a minimal PATH (~/usr/local/bin:/usr/bin:/bin). +# `uv` and `claude` are installed under the runtime user's `~/.local/bin` +# so we have to prepend it explicitly; otherwise run_daily.sh fails at +# the up-front command-presence check. +Environment=PATH=/home/mateo/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +# `HOME` is auto-set to /home/mateo when User=mateo is honored, but be +# explicit so anything that reads $HOME (e.g. uv's cache lookup, the +# claude CLI's per-session dir) sees the right value even if a future +# refactor flips DynamicUser= or PrivateUsers= on. +Environment=HOME=/home/mateo + +WorkingDirectory=/home/mateo/litellm/litellm + +ExecStart=/home/mateo/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh + +# 90 minutes is generous: cold runs do `git clone` + `uv sync` of a new +# tag's lockfile, which can take a couple of minutes on a 2-vCPU VM, +# plus 30 cells of pytest hitting four cloud providers. +TimeoutStartSec=90min + +# A failed run shouldn't restart automatically — the next timer fire is +# the right retry. Reruns of the same day's matrix are idempotent. +Restart=no + +# Security hardening: the populator only reads the litellm checkout and +# the env-file; everything else it writes lives in either the worktree +# (managed) or `/tmp` (cleaned up by tempfile). +# +# `ProtectHome=read-only` blocks writes to /home/mateo but still +# allows reads. That's safe for the trusted run_daily.sh script +# itself, but unsafe for any subprocess we don't control: a +# compromised npm-installed `claude` package, or a model-directed +# `Read` tool call during a PDF/vision cell, could read sensitive +# host files like `~/.config/gh/hosts.yml` (gh-host token), +# `~/.ssh/`, or `~/.bash_history`. We mitigate that at the call +# boundary: every `claude` subprocess (the up-front `claude --version` +# probe in run_daily.sh, plus every CLI invocation routed through +# tests/e2e/claude_code/cli_driver.py) runs with `HOME` pointed at a +# fresh empty per-invocation tmpdir, not at /home/mateo. The CLI +# never sees the runtime user's real dotfiles. `gh` invocations in +# run_daily.sh pass `GH_TOKEN` inline, so they never need to read +# ~/.config/gh either; that path is intentionally NOT in the +# whitelist below — keeping it out is the second line of defense if +# the inline-token convention is ever accidentally regressed. +# +# ReadWritePaths whitelist: +# * litellm-cron-worktree - the long-lived stable-tag checkout + +# its `.venv` (`uv sync` rewrites every +# run) + `.uv-bin` (pinned `uv` binary +# cache). +# * .cache - uv's wheel cache (~/.cache/uv) so we +# don't redownload pinned deps each +# run. Used only by the trusted `uv` +# process; not exposed to `claude`. +# * /tmp - mktemp -d workdir, proxy logs, and +# the per-`claude`-invocation isolated +# HOME tmpdirs. PrivateTmp=true below +# gives the service its own tmpfs view +# so these don't escape to the host. +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths=/home/mateo/litellm-cron-worktree /home/mateo/.cache /tmp +PrivateTmp=true + +# Filesystem-level hiding for credential-bearing dotdirs/files. Even +# though `ProtectHome=read-only` prevents writes, a model-directed +# `Read` tool call (the PDF cells pass `--allowed-tools Read`) or a +# compromised `claude` package can read absolute paths under +# /home/mateo and exfiltrate the contents. `InaccessiblePaths=` makes +# the listed paths look like empty/missing to every process in the +# unit's mount namespace -- including the trusted populator script, +# which is fine because it doesn't need any of these: +# +# * .config/gh - gh CLI host token; we pass GH_TOKEN inline to +# every `gh` invocation (clone/PR/reviewer) so the +# host config is never consulted. +# * .ssh - never used by the populator. +# * .aws - upstream AWS credentials are passed to the proxy +# via the EnvironmentFile (provider env vars), not +# via shared SDK config files. +# * .docker - the populator never talks to a docker socket. +# * .kube - the populator never talks to a k8s API. +# * .gnupg - no GPG signing on the bot's commits. +# +# Leading `-` makes systemd tolerant if a path doesn't exist on the +# host (the unit is portable across VMs that may not have all of +# them set up). Anything else under /home/mateo (the litellm +# checkout, the cron worktree, the uv cache, .local/bin for the +# claude/uv/gh binaries on PATH) stays read-accessible. +InaccessiblePaths=-/home/mateo/.config/gh -/home/mateo/.ssh -/home/mateo/.aws -/home/mateo/.docker -/home/mateo/.kube -/home/mateo/.gnupg + +[Install] +WantedBy=multi-user.target diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer new file mode 100644 index 00000000000..ee22538c6ed --- /dev/null +++ b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer @@ -0,0 +1,25 @@ +# Daily timer for the compatibility-matrix populator. +# +# 06:00 UTC matches the original GitHub Actions cron schedule; chosen so +# operators in US/EU timezones see fresh PRs at the start of their work +# day. +# +# `Persistent=true` causes a missed run (VM was off / suspended) to +# fire the next time the timer is started, which is the property we +# want for a once-a-day job: the matrix should refresh as soon as the +# VM is reachable again, not wait another 24h. +# +# `RandomizedDelaySec=10min` smears load if multiple matrix-style +# pipelines are ever colocated on the same VM in the future. + +[Unit] +Description=Run the Claude Code compatibility-matrix populator daily + +[Timer] +OnCalendar=*-*-* 06:00:00 UTC +Persistent=true +RandomizedDelaySec=10min +Unit=litellm-compat-matrix.service + +[Install] +WantedBy=timers.target diff --git a/tests/e2e/claude_code/cron_vm/run_daily.sh b/tests/e2e/claude_code/cron_vm/run_daily.sh new file mode 100755 index 00000000000..ae2d67c070c --- /dev/null +++ b/tests/e2e/claude_code/cron_vm/run_daily.sh @@ -0,0 +1,590 @@ +#!/usr/bin/env bash +# Daily Claude Code compatibility-matrix populator. +# +# Runs from the GCP VM `litellm-compatibility-matrix-populator` via the +# systemd timer in this directory. The flow is: +# +# 1. Resolve the latest LiteLLM v*-stable tag from the GitHub Releases API. +# 2. Update a long-lived worktree at $WORKTREE to that tag and `uv sync` it. +# 3. Boot the proxy as a background subprocess on $PROXY_PORT (default +# 4100; a separate port from the human-tended :4000 proxy). +# 4. Run `pytest tests/e2e/claude_code/` against the proxy. Test failures +# become `fail` cells in the JSON, not script errors. +# 5. Hand the per-test results artifact + manifest to a small Python +# CLI (`build_matrix.py`) that wraps the existing +# `matrix_builder.build_from_paths` to produce the published +# compatibility-matrix.json. +# 6. `gh repo clone` litellm-docs, write the JSON to a deterministic +# branch (`compat-matrix/--`), commit, +# `git push --force`, and `gh pr create`. +# +# Same-day reruns land on the same branch so they update the existing PR +# rather than spawning a new one. If the JSON is byte-identical to the +# docs branch, we skip the push entirely. +# +# Required commands on $PATH: git, uv, gh, jq, curl, claude. +# Required state: ~/litellm/litellm checked out (this file lives in it), +# $WORKTREE is created on first run, gh is already authenticated. +# +# Override any default by setting the matching env var; see the systemd +# unit for the production wiring. + +set -Eeuo pipefail + +LITELLM_REPO="${LITELLM_REPO:-${HOME}/litellm/litellm}" +WORKTREE="${LITELLM_WORKTREE:-${HOME}/litellm-cron-worktree}" +PROXY_PORT="${PROXY_PORT:-4100}" +PROXY_API_KEY="${PROXY_API_KEY:-sk-cron-matrix}" +DOCS_REPO="${DOCS_REPO:-BerriAI/litellm-docs}" +DOCS_BRANCH="${DOCS_BRANCH:-main}" +DOCS_TARGET_PATH="${DOCS_TARGET_PATH:-src/data/compatibility-matrix.json}" +SKIP_PUBLISH="${SKIP_PUBLISH:-0}" +PYTEST_K="${PYTEST_K:-}" +# Comma-separated GitHub usernames to request a review from on every PR. +# Reviewers must have at least read access to ${DOCS_REPO}. PR-author +# (agent-shin) has implicit rights to request reviews from anyone with +# read access, so no extra token scope is needed. Set to empty to skip. +PR_REVIEWERS="${PR_REVIEWERS:-mateo-berri}" + +POPULATOR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORKDIR="$(mktemp -d -t litellm-compat-matrix.XXXXXX)" +PROXY_PID_FILE="${WORKDIR}/proxy.pid" + +# Cleanup is intentionally aggressive: it can run on normal exit, on a +# signal received by the script, or after a partial failure where the +# proxy is up but ${PROXY_PID_FILE} is stale. We try four things in +# order and stop as soon as the proxy port is free: +# +# 1. SIGTERM the pid recorded in proxy.pid. +# 2. SIGKILL anything from `pgrep -f "litellm.*--port ${PROXY_PORT}"` +# that survived. This catches the common case where the recorded +# pid was the sh wrapper, not the long-lived python child. +# 3. ss -K on the port (kernel kills sockets but not processes; +# mostly useful for catching lingering CLOSE_WAITs). +# 4. wipe ${WORKDIR}. +cleanup() { + local rc=$? + set +e + local proxy_pid + if [[ -f "${PROXY_PID_FILE}" ]]; then + proxy_pid="$(cat "${PROXY_PID_FILE}")" + if [[ -n "${proxy_pid}" ]]; then + kill -TERM "-${proxy_pid}" 2>/dev/null || kill -TERM "${proxy_pid}" 2>/dev/null || true + for _ in 1 2 3 4 5; do + kill -0 "${proxy_pid}" 2>/dev/null || break + sleep 1 + done + fi + fi + # Belt-and-braces: any python or uv talking to ${PROXY_PORT} that + # survived the SIGTERM gets SIGKILL'd by name. + pgrep -f "litellm.*--port[ =]?${PROXY_PORT}([^0-9]|$)" 2>/dev/null \ + | xargs -r kill -KILL 2>/dev/null || true + pgrep -f "${WORKTREE}/.uv-bin/uv.*run litellm" 2>/dev/null \ + | xargs -r kill -KILL 2>/dev/null || true + rm -rf "${WORKDIR}" + exit "${rc}" +} +trap cleanup EXIT INT TERM + +log() { printf '==> %s\n' "$*" >&2; } +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } + +for cmd in git uv gh jq curl claude; do + command -v "${cmd}" >/dev/null 2>&1 || die "missing required command: ${cmd}" +done + +# Publishing is from a fork (agent-shin/litellm-docs) so neither the cron +# host nor the bot identity needs write access to BerriAI/litellm-docs. We +# require the fork token up front -- failing 30 minutes into a run because +# the env file is missing one line is a waste of CI quota. +if [[ "${SKIP_PUBLISH}" != "1" ]]; then + [[ -n "${AGENT_SHIN_GITHUB_TOKEN:-}" ]] \ + || die "AGENT_SHIN_GITHUB_TOKEN required to open PRs from agent-shin/litellm-docs (or set SKIP_PUBLISH=1)" +fi + +# --------------------------------------------------------------------------- +# 1. Resolve versions +# --------------------------------------------------------------------------- + +# Newest v*-stable release on BerriAI/litellm. The `select(...)` filter +# drops drafts/non-stable, the version_key sort handles 1.10 > 1.9. +# +# Paginate through the releases endpoint instead of grabbing only page 1 +# (default page_size=30). LiteLLM ships multiple non-stable releases per +# day, so it's common to need to walk past 30+ entries before hitting +# the most recent v*-stable. We cap at 5 pages (500 releases) which is +# conservatively beyond the worst observed gap. +# +# We deliberately do NOT short-circuit on the first page that contains a +# v*-stable tag. The /releases endpoint orders by `created_at`, not by +# semver, so a backport on an older series (e.g. v1.80.1-stable cut +# today) can show up on an earlier page than a higher-versioned release +# (v1.83.0-stable cut two weeks ago). Breaking early on first-stable-seen +# would silently pin the cron to the stale tag because the +# higher-versioned release still on a later page would never make it +# into the merged set the `sort_by` below consumes. The only break we +# keep is the empty-page guard, which means a quiet period in the +# release feed doesn't waste API quota — we just always walk far enough +# to be confident we've seen the highest stable tag. +GH_AUTH_HEADER=() +if [[ -n "${GITHUB_TOKEN:-}" ]]; then + GH_AUTH_HEADER=(-H "Authorization: Bearer ${GITHUB_TOKEN}") +fi +RELEASES_JSON="${WORKDIR}/releases.json" +echo "[]" >"${RELEASES_JSON}" +for page in 1 2 3 4 5; do + PAGE_JSON="${WORKDIR}/releases.page${page}.json" + curl -fsS \ + -H 'Accept: application/vnd.github+json' \ + -H 'User-Agent: litellm-compat-matrix' \ + "${GH_AUTH_HEADER[@]}" \ + "https://api.github.com/repos/BerriAI/litellm/releases?per_page=100&page=${page}" \ + >"${PAGE_JSON}" + jq -s '.[0] + .[1]' "${RELEASES_JSON}" "${PAGE_JSON}" >"${RELEASES_JSON}.merged" + mv "${RELEASES_JSON}.merged" "${RELEASES_JSON}" + # No more pages? GitHub returns an empty array past the last page. + if [[ "$(jq 'length' "${PAGE_JSON}")" == "0" ]]; then + break + fi +done +LITELLM_VERSION="$( + jq -r ' + [ .[] | .tag_name // empty + | select(test("^v[0-9]+\\.[0-9]+\\.[0-9]+-stable$")) + ] + | sort_by( + capture("^v(?[0-9]+)\\.(?[0-9]+)\\.(?[0-9]+)-stable$") + | [(.a|tonumber), (.b|tonumber), (.c|tonumber)] + ) + | last // empty + ' "${RELEASES_JSON}" +)" +[[ -n "${LITELLM_VERSION}" ]] || die "could not resolve latest v*-stable tag in 5 pages of releases" +log "resolved litellm: ${LITELLM_VERSION}" + +# The systemd unit loads provider credentials and the agent-shin GitHub +# token from /etc/litellm-compat-matrix.env into this script's +# environment. Running the npm-installed `claude` binary directly here +# would hand that full env to package code -- a compromised +# @anthropic-ai/claude-code release could read ANTHROPIC_API_KEY / +# AWS_BEARER_TOKEN_BEDROCK / AZURE_FOUNDRY_API_KEY / +# AGENT_SHIN_GITHUB_TOKEN from os.environ and exfiltrate them before +# the proxy or test harness ever starts. Probe under `env -i` with the +# same minimal allowlist the PR-gate uses (the matrix run itself goes +# through cli_driver.py, which already scrubs the CLI env). +# +# The probe also runs under a fresh empty HOME instead of the runtime +# user's real $HOME. `ProtectHome=read-only` in the systemd unit +# blocks *writes* to /home/mateo but still allows reads, so a +# compromised claude package invoked here with HOME=/home/mateo could +# read ~/.config/gh/hosts.yml (the gh-host token), ~/.bash_history, +# or ~/.ssh/. Pointing HOME at a per-run dir under ${WORKDIR} hides +# those entirely from the subprocess; ${WORKDIR} is rm -rf'd by the +# script-wide cleanup() trap regardless of probe outcome. +CLAUDE_PROBE_HOME="${WORKDIR}/claude-probe-home" +mkdir -p "${CLAUDE_PROBE_HOME}" +CLAUDE_CODE_VERSION="$(env -i \ + PATH="${PATH}" \ + HOME="${CLAUDE_PROBE_HOME}" \ + USER="${USER:-mateo}" \ + TERM="${TERM:-dumb}" \ + LANG="${LANG:-C.UTF-8}" \ + LC_ALL="${LC_ALL:-}" \ + TMPDIR="${TMPDIR:-/tmp}" \ + claude --version 2>/dev/null \ + | grep -oE '[0-9]+\.[0-9]+\.[0-9]+([.-][A-Za-z0-9.-]+)?' \ + | head -n1 || true)" +# `|| true` above keeps `set -Eeuo pipefail` from aborting silently when +# `grep` finds no match (exit 1) — without it the assignment inherits the +# pipeline's non-zero exit, `set -e` kills the script, and the operator +# never sees the helpful diagnostic below. +[[ -n "${CLAUDE_CODE_VERSION}" ]] || die "could not parse semver from 'claude --version'" +log "local claude code: ${CLAUDE_CODE_VERSION}" + +# --------------------------------------------------------------------------- +# 2. Update the worktree to that tag +# --------------------------------------------------------------------------- + +if [[ ! -d "${WORKTREE}/.git" ]]; then + log "first run: cloning litellm into ${WORKTREE}" + mkdir -p "$(dirname "${WORKTREE}")" + git clone https://github.com/BerriAI/litellm.git "${WORKTREE}" +fi + +log "updating worktree to ${LITELLM_VERSION}" +git -C "${WORKTREE}" fetch --tags --force +git -C "${WORKTREE}" reset --hard +# Keep the venv and the .uv-bin cache around — uv sync will reconcile +# the venv on every run, and we don't want to re-download the pinned +# uv binary each time. Drop everything else (including any prior +# tests/e2e/claude_code/ shim) so each run starts clean before the shim +# below rewrites it from the dev checkout. +git -C "${WORKTREE}" clean -fdx -e .venv -e .uv-bin +git -C "${WORKTREE}" checkout --force "${LITELLM_VERSION}" + +# Always overwrite tests/e2e/claude_code/ in the worktree with the copy +# from the dev checkout, regardless of whether the resolved +# ${LITELLM_VERSION} tag already ships a tests/e2e/claude_code/ tree of +# its own. Rationale: the matrix populator's job is to exercise +# today's tests against the latest stable proxy. The dev checkout +# carries the most recent test fixes (e.g. the stream-json vision +# rewrite, the --effort thinking knob, the WebSearch tool_use +# assertion) that haven't yet rolled into a v*-stable, and we want +# every cron run to pick those up the moment they land on +# ${LITELLM_REPO}, not whenever the next stable release happens. +# +# Concretely this means a fresh `rm -rf` + `cp -r` every run so the +# tree is byte-identical to ${LITELLM_REPO}/tests/e2e/claude_code (no +# stale files left over from the tag's own checkout, no drift across +# runs). +if [[ ! -d "${LITELLM_REPO}/tests/e2e/claude_code" ]]; then + die "no shim source at ${LITELLM_REPO}/tests/e2e/claude_code" +fi +log "shimming tests/e2e/claude_code/ from ${LITELLM_REPO} (always-overwrite)" +rm -rf "${WORKTREE}/tests/e2e/claude_code" +mkdir -p "${WORKTREE}/tests/e2e" +cp -r "${LITELLM_REPO}/tests/e2e/claude_code" "${WORKTREE}/tests/e2e/" + +# litellm pins an exact uv version in pyproject.toml's [tool.uv] +# `required-version` field, so a system uv that's newer or older +# refuses to sync. We pin our own local copy at the version the +# checked-out tag asks for, cached under .uv-bin/ inside the worktree +# so subsequent runs skip the download. +PINNED_UV_VERSION="$( + awk -F'"' ' + /^required-version[[:space:]]*=/ { + # Field 2 is the value between the quotes, e.g. ">=0.10.9" or + # "0.10.9". Strip any leading specifier prefix so we end up with + # the bare version string, which is what /releases/download// + # expects. + v = $2 + sub(/^[[:space:]=<>!~]+/, "", v) + if (v != "") { print v; exit } + } + ' "${WORKTREE}/pyproject.toml" +)" +if [[ -z "${PINNED_UV_VERSION}" ]]; then + log "no uv version pin in pyproject.toml; using system uv" + WORKTREE_UV="$(command -v uv)" +else + WORKTREE_UV="${WORKTREE}/.uv-bin/uv-${PINNED_UV_VERSION}" + if [[ ! -x "${WORKTREE_UV}" ]]; then + log "downloading uv ${PINNED_UV_VERSION} for the worktree" + mkdir -p "${WORKTREE}/.uv-bin" + # Detect host arch so the same script works on x86_64 GCP VMs and on + # aarch64 hosts (Astral publishes both `uv-x86_64-unknown-linux-gnu` + # and `uv-aarch64-unknown-linux-gnu` tarballs under the same release + # tag, and `uname -m` already returns the exact token uv uses). + UV_ARCH="$(uname -m)" + UV_TRIPLE="uv-${UV_ARCH}-unknown-linux-gnu" + UV_TARBALL_NAME="${UV_TRIPLE}.tar.gz" + UV_DOWNLOAD_URL="https://github.com/astral-sh/uv/releases/download/${PINNED_UV_VERSION}/${UV_TARBALL_NAME}" + UV_TMPDIR="$(mktemp -d -t uv-download.XXXXXX)" + # Download the tarball and Astral's official .sha256 sidecar to disk + # and verify the digest before extracting/executing anything. This + # closes the supply-chain trust gap of piping a remote binary + # straight into `tar -xzO ... > file ; chmod +x` (see CLAUDE.md + # "CI Supply-Chain Safety"). + curl -fsSL --output "${UV_TMPDIR}/${UV_TARBALL_NAME}" "${UV_DOWNLOAD_URL}" + curl -fsSL --output "${UV_TMPDIR}/${UV_TARBALL_NAME}.sha256" "${UV_DOWNLOAD_URL}.sha256" + (cd "${UV_TMPDIR}" && sha256sum -c "${UV_TARBALL_NAME}.sha256") \ + || { rm -rf "${UV_TMPDIR}"; die "uv ${PINNED_UV_VERSION} sha256 mismatch — refusing to install"; } + tar -xzf "${UV_TMPDIR}/${UV_TARBALL_NAME}" -C "${UV_TMPDIR}" "${UV_TRIPLE}/uv" + mv "${UV_TMPDIR}/${UV_TRIPLE}/uv" "${WORKTREE_UV}.tmp" + chmod +x "${WORKTREE_UV}.tmp" + mv "${WORKTREE_UV}.tmp" "${WORKTREE_UV}" + rm -rf "${UV_TMPDIR}" + fi +fi +# `--extra proxy` pulls fastapi/uvicorn/etc. so `uv run litellm` can +# actually serve. `--group proxy-dev` brings in pytest and the rest of +# what tests/e2e/claude_code/ needs. +log "uv sync --frozen --group proxy-dev --extra proxy (uv ${PINNED_UV_VERSION:-system})" +(cd "${WORKTREE}" && "${WORKTREE_UV}" sync --frozen --group proxy-dev --extra proxy) + +PROXY_CONFIG="${WORKTREE}/tests/e2e/claude_code/test_config.yaml" +[[ -f "${PROXY_CONFIG}" ]] || die "proxy config not found at ${PROXY_CONFIG} (does ${LITELLM_VERSION} predate the compat matrix work?)" + +# --------------------------------------------------------------------------- +# 3. Boot the proxy +# --------------------------------------------------------------------------- + +log "starting proxy on 127.0.0.1:${PROXY_PORT}" +# Bind the proxy to loopback only. The populator proxy is talked to +# exclusively by the pytest run on the same host (the health check and +# the test env set `LITELLM_PROXY_BASE_URL=http://127.0.0.1:...`), +# so there's no reason to expose it on the VM's external interfaces. +# Without `--host`, `litellm` defaults to 0.0.0.0, which combined with +# the predictable default `LITELLM_MASTER_KEY=sk-cron-matrix` would +# allow anything that can reach :${PROXY_PORT} on the VM to authenticate +# and burn upstream provider credentials. +# +# `setsid` puts the proxy in its own session+pgroup so cleanup() can +# SIGTERM the whole tree by passing the pgid as a negative pid. We +# write that pid to a file so cleanup() doesn't need to remember a +# variable that might be stale by the time the trap fires. +# +# Pass the master key as a shell-prefix assignment on `setsid` (inherited +# via the environment) rather than as `env KEY=VAL ...` argv. The argv +# form would land the literal key in /proc//cmdline, where +# any local reader (a model-directed `Read` tool call, another user on +# the VM, a crash dump) could pick it up before the process execs into +# the litellm child. The shell-prefix form keeps the key out of argv at +# every layer (setsid → bash → uv → litellm). +LITELLM_MASTER_KEY="${PROXY_API_KEY}" setsid bash -c ' + echo "$$" > "$0" + cd "$1" + exec "$2" run litellm --config "$3" --host 127.0.0.1 --port "$4" +' "${PROXY_PID_FILE}" "${WORKTREE}" "${WORKTREE_UV}" "${PROXY_CONFIG}" "${PROXY_PORT}" \ + >"${WORKDIR}/proxy.log" 2>&1 & +disown + +HEALTH_URL="http://127.0.0.1:${PROXY_PORT}/health/liveliness" +for _ in $(seq 1 45); do + if curl -fsS "${HEALTH_URL}" >/dev/null 2>&1; then + break + fi + sleep 2 +done +curl -fsS "${HEALTH_URL}" >/dev/null \ + || { tail -50 "${WORKDIR}/proxy.log" >&2; die "proxy did not become healthy"; } + +# --------------------------------------------------------------------------- +# 4. Run pytest +# --------------------------------------------------------------------------- + +RESULTS_JSON="${WORKDIR}/compat-results.json" +PYTEST_ARGS=( + tests/e2e/claude_code/ + --ignore=tests/e2e/claude_code/_driver_unit_tests + --ignore=tests/e2e/claude_code/_builder_unit_tests + --ignore=tests/e2e/claude_code/_publisher_unit_tests + --ignore=tests/e2e/claude_code/_pr_gate_unit_tests +) +if [[ -n "${PYTEST_K}" ]]; then + log "PYTEST_K set; narrowing to: ${PYTEST_K}" + PYTEST_ARGS+=(-k "${PYTEST_K}") +fi + +log "running pytest" +set +e +# Pytest only needs to talk to the loopback proxy at 127.0.0.1:${PROXY_PORT} +# — it has no legitimate reason to see ANTHROPIC_API_KEY / +# AWS_BEARER_TOKEN_BEDROCK / VERTEXAI_* / AZURE_FOUNDRY_* / +# AGENT_SHIN_GITHUB_TOKEN / GITHUB_TOKEN in its own env. The systemd +# unit's EnvironmentFile injects all of those into this script for the +# proxy to consume, and pytest inherits them by default. Wrap the +# invocation in `env -i` so: +# +# 1. test code under tests/e2e/claude_code/ (or anything it imports) +# cannot read provider/agent-shin creds out of `os.environ` and +# exfiltrate them via an outbound call from inside a conftest hook +# or a fixture (a sibling vector to the model-controlled Bash/Read +# concern handled by `cli_driver.py`'s own env scrub); +# 2. a model-directed `Read` tool call during a PDF/vision cell +# cannot reach /proc//environ and pull the creds out +# of the parent process the way it can today; +# 3. this matches the PR-gate pytest step in `.circleci/config.yml`, +# which already runs under `env -i` with the same minimal +# allowlist. +# +# `cli_driver.py` re-allowlists its own subset (PATH/USER/LOGNAME/etc.) +# when spawning the `claude` binary, so the CLI still finds Node + the +# claude shim on PATH and gets a fresh isolated HOME per invocation. +( + cd "${WORKTREE}" \ + && env -i \ + PATH="${PATH}" \ + HOME="${HOME}" \ + USER="${USER:-mateo}" \ + TERM="${TERM:-dumb}" \ + LANG="${LANG:-C.UTF-8}" \ + LC_ALL="${LC_ALL:-}" \ + TMPDIR="${TMPDIR:-/tmp}" \ + LITELLM_PROXY_BASE_URL="http://127.0.0.1:${PROXY_PORT}" \ + LITELLM_PROXY_API_KEY="${PROXY_API_KEY}" \ + COMPAT_RESULTS_PATH="${RESULTS_JSON}" \ + "${WORKTREE_UV}" run pytest "${PYTEST_ARGS[@]}" +) +PYTEST_EXIT=$? +set -e +log "pytest exit code: ${PYTEST_EXIT} (failures become 'fail' cells, not script errors)" +[[ -f "${RESULTS_JSON}" ]] || die "pytest did not produce ${RESULTS_JSON}" + +# --------------------------------------------------------------------------- +# 5. Build the matrix JSON +# --------------------------------------------------------------------------- + +MATRIX_JSON="${WORKDIR}/compatibility-matrix.json" +log "building ${MATRIX_JSON}" +( + cd "${WORKTREE}" \ + && "${WORKTREE_UV}" run python "${POPULATOR_DIR}/build_matrix.py" \ + --manifest "${WORKTREE}/tests/e2e/claude_code/manifest.yaml" \ + --results "${RESULTS_JSON}" \ + --output "${MATRIX_JSON}" \ + --litellm-version "${LITELLM_VERSION}" \ + --claude-code-version "${CLAUDE_CODE_VERSION}" +) + +# --------------------------------------------------------------------------- +# 6. Open a docs-repo PR +# --------------------------------------------------------------------------- + +if [[ "${SKIP_PUBLISH}" == "1" ]]; then + cp "${MATRIX_JSON}" "${LITELLM_REPO}/compatibility-matrix.json" + log "SKIP_PUBLISH=1; matrix written to ${LITELLM_REPO}/compatibility-matrix.json" + exit 0 +fi + +DATE_UTC="$(date -u +%Y-%m-%d)" +BRANCH_NAME="compat-matrix/${LITELLM_VERSION}-${CLAUDE_CODE_VERSION}-${DATE_UTC}" +DOCS_CLONE="${WORKDIR}/litellm-docs" +FORK_OWNER="${FORK_OWNER:-agent-shin}" +FORK_REPO="${FORK_REPO:-${FORK_OWNER}/litellm-docs}" + +log "cloning ${DOCS_REPO}@${DOCS_BRANCH}" +# Use the agent-shin token inline rather than the host gh-cli config. +# `BerriAI/litellm-docs` is a public repo so unauthenticated clone +# would also work, but passing the token explicitly means the systemd +# unit can hide `~/.config/gh` (`InaccessiblePaths=`) without breaking +# this clone — closing the model-directed `Read("/home/mateo/.config/gh/...")` +# exfiltration path on the cron VM. +GH_TOKEN="${AGENT_SHIN_GITHUB_TOKEN}" \ + gh repo clone "${DOCS_REPO}" "${DOCS_CLONE}" -- --depth 1 --branch "${DOCS_BRANCH}" + +cd "${DOCS_CLONE}" +git config user.email "litellm-bot@berri.ai" +git config user.name "litellm-compat-matrix-bot" +git checkout -b "${BRANCH_NAME}" + +mkdir -p "$(dirname "${DOCS_TARGET_PATH}")" +cp "${MATRIX_JSON}" "${DOCS_TARGET_PATH}" +git add "${DOCS_TARGET_PATH}" + +if git diff --cached --quiet; then + log "matrix JSON unchanged from ${DOCS_BRANCH}; skipping PR" + exit 0 +fi + +GENERATED_AT="$(jq -r '.generated_at' "${MATRIX_JSON}")" +COMMIT_MSG="$(cat </dev/null || true +git remote add fork "${FORK_PUSH_URL}" +git push --force --set-upstream fork "${BRANCH_NAME}" +git remote remove fork +unset FORK_PUSH_URL + +# Per-feature status table for the PR body. Reviewers triage from this. +PR_FEATURE_TABLE="$(jq -r ' + .features[] as $f + | "- **\($f.name)**: " + + ([ .providers[] as $p + | "\($p)=\($f.providers[$p].status // "not_tested")" + ] | join(", ")) +' "${MATRIX_JSON}")" + +PR_TITLE="chore(compat-matrix): refresh for ${LITELLM_VERSION} + claude-code ${CLAUDE_CODE_VERSION}" +PR_BODY="$(cat < ${DOCS_REPO}:${DOCS_BRANCH}" +# GH_TOKEN here is scoped to this single subshell so we don't bleed the +# fork token into the rest of the script (release-listing earlier uses +# ${GITHUB_TOKEN}, which may be a different identity). gh's --head accepts +# `OWNER:BRANCH` for cross-repo PRs from a fork. +# +# Reviewer assignment is done in a *separate* call below: as the PR +# author from a fork, agent-shin has no write/triage access on +# ${DOCS_REPO} and the `RequestReviewsByLogin` GraphQL mutation +# (which backs `gh pr create --reviewer` and `gh pr edit --add-reviewer`) +# rejects with "does not have the correct permissions". We use the +# collaborator-scoped ${GITHUB_TOKEN} for that instead. Don't fold +# --reviewer into `gh pr create` here -- it would fail the whole +# create on the very first cron run. +set +e +PR_OUT="$( + GH_TOKEN="${AGENT_SHIN_GITHUB_TOKEN}" gh pr create \ + --repo "${DOCS_REPO}" \ + --base "${DOCS_BRANCH}" \ + --head "${FORK_OWNER}:${BRANCH_NAME}" \ + --title "${PR_TITLE}" \ + --body "${PR_BODY}" 2>&1 +)" +PR_EXIT=$? +set -e +echo "${PR_OUT}" + +if [[ ${PR_EXIT} -ne 0 ]]; then + if grep -q "a pull request for branch.*already exists" <<<"${PR_OUT}"; then + log "PR already exists for ${FORK_OWNER}:${BRANCH_NAME}; updated branch in place" + else + die "gh pr create failed (exit ${PR_EXIT})" + fi +fi + +# Request reviews from PR_REVIEWERS using the collaborator-scoped +# ${GITHUB_TOKEN} (mateo-berri's token, already provisioned for release +# listing). This is idempotent: `gh pr edit --add-reviewer` is a no-op +# on a user who's already in reviewRequests, and silently re-adds +# anyone whose prior review was dismissed -- so same-day reruns stay +# clean. Reviewer-add failures are non-fatal: the matrix JSON has +# already landed on the PR; the worst case is a manual ping. +if [[ -n "${PR_REVIEWERS}" ]]; then + if [[ -z "${GITHUB_TOKEN:-}" ]]; then + log "WARN: PR_REVIEWERS set but GITHUB_TOKEN missing -- cannot request reviews; skipping" + else + log "requesting reviews from: ${PR_REVIEWERS}" + set +e + GH_TOKEN="${GITHUB_TOKEN}" gh pr edit \ + "${FORK_OWNER}:${BRANCH_NAME}" \ + --repo "${DOCS_REPO}" \ + --add-reviewer "${PR_REVIEWERS}" 2>&1 | sed 's/^/ /' + REVIEWER_EXIT=${PIPESTATUS[0]} + set -e + if [[ ${REVIEWER_EXIT} -ne 0 ]]; then + log "WARN: gh pr edit --add-reviewer exited ${REVIEWER_EXIT} (non-fatal)" + fi + fi +fi + +log "done" diff --git a/tests/e2e/claude_code/http_probe.py b/tests/e2e/claude_code/http_probe.py new file mode 100644 index 00000000000..d95307db3b9 --- /dev/null +++ b/tests/e2e/claude_code/http_probe.py @@ -0,0 +1,292 @@ +"""Direct HTTP probe helpers for the Claude Code compatibility matrix. + +Most matrix cells drive the `claude` CLI in headless mode and observe +the stream-json wire (see `cli_driver.py`). A handful of features the +proxy must support don't have any CLI surface area -- `count_tokens` is +the canonical example: Claude Code calls it internally for budget +display, but the result never appears in stream-json events, so a CLI +test cannot observe whether the endpoint round-tripped correctly +through the proxy for any given provider. + +This module is the second test pattern the matrix supports: a plain +HTTP POST against a LiteLLM proxy endpoint, parsed and shape-checked +in the test, with the same `compat_result` recording convention as the +CLI-driven cells. The goal is to keep this pattern *narrow* -- if a +feature can be tested via the CLI, it should be, because the CLI path +is closer to what real Claude Code users hit. HTTP probes are only for +features the CLI can't reach. + +The probe deliberately uses a short timeout (30s) and small payloads: +this is a "did the request shape survive the proxy's +provider-specific transformations" test, not a load test, and a real +endpoint regression typically surfaces in well under a second of wall +time (400 / 500 from the upstream, or LiteLLM 500 on a transformation +bug). +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any, Mapping, Optional + +import httpx + +from claude_code.rate_limiter import ( + RateLimiter, + get_default_limiter, + infer_provider, +) + + +DEFAULT_TIMEOUT_SECONDS = 30.0 + + +@dataclass +class ProbeResult: + """Structured outcome of a single HTTP probe. + + `status_code` and `body` are the wire response; `payload` is the + parsed JSON body if the response was JSON, else None. Tests assert + on `status_code` + `payload` shape; `body` is preserved so failure + diagnostics can echo the raw error string (which is the only thing + a maintainer needs to triage a red cell). + """ + + status_code: int + body: str + payload: Optional[Mapping[str, Any]] = None + error: Optional[str] = None + + +def probe_count_tokens( + *, + base_url: str, + api_key: str, + model: str, + message: str = "hello world", + timeout: float = DEFAULT_TIMEOUT_SECONDS, + rate_limiter: Optional[RateLimiter] = None, +) -> ProbeResult: + """POST to `{base_url}/v1/messages/count_tokens` for `model` and return the parsed result. + + The Anthropic / LiteLLM `count_tokens` endpoint accepts a request + body whose shape mirrors `/v1/messages` (model + messages), and + returns `{"input_tokens": N}` for a successful response. Anything + else -- non-200 status, non-JSON body, missing/non-int + `input_tokens` -- is a regression we want the cell to flip red on. + + The same cross-process token-bucket limiter `cli_driver.run_claude` + uses is acquired here too, so probe rows count against the + aggregate per-provider budget. Without this, an HTTP-probe row + would fire unthrottled requests in parallel with throttled CLI + rows and silently violate the limiter's aggregate-rate guarantee. + `rate_limiter` is an injection seam for unit tests; production + callers should leave it unset to use the process-wide default. + """ + limiter = rate_limiter if rate_limiter is not None else get_default_limiter() + limiter.acquire(infer_provider(model)) + + url = base_url.rstrip("/") + "/v1/messages/count_tokens" + try: + response = httpx.post( + url, + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + # `anthropic-version` is required by Anthropic's native + # API and harmless on every other provider the proxy + # routes to. Matches what the Claude Code CLI sends + # for its own internal `count_tokens` calls. + "anthropic-version": "2023-06-01", + }, + json={"model": model, "messages": [{"role": "user", "content": message}]}, + timeout=timeout, + ) + except httpx.HTTPError as exc: + return ProbeResult(status_code=0, body="", error=f"transport: {exc}") + + body = response.text or "" + try: + payload = response.json() if body else None + except (json.JSONDecodeError, ValueError): + payload = None + + return ProbeResult( + status_code=response.status_code, + body=body, + payload=payload, + ) + + +def probe_tool_search( + *, + base_url: str, + api_key: str, + model: str, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + rate_limiter: Optional[RateLimiter] = None, +) -> ProbeResult: + """POST to `{base_url}/v1/messages` with a `tool_search_tool_regex_20251119` + tool definition and return the result. + + The shape of the tools array is the one Claude Code emits when its + MCP-tool-search beta is active: a `tool_search_tool_regex_20251119` + discovery tool (name `tool_search_tool_regex`) plus at least one + regular user tool to be searched. LiteLLM's + `is_tool_search_used` helper keys on the `_20251119`-suffixed type + string to decide whether to attach the provider-specific tool-search + beta header (`advanced-tool-use-2025-11-20` for Anthropic/Azure, + `tool-search-tool-2025-10-19` for Vertex/Bedrock). A proxy + regression in that translation will surface here as a 400 from + the upstream complaining about the tool type or beta header. + + The prompt deliberately does not force a tool call -- the goal is + to verify the *request* round-trips without 400 and produces some + response, not to test whether the model decided to invoke + tool_search. That kind of behavior test would couple this row to + Claude Code's model behavior heuristics, which change weekly. + + Like `probe_count_tokens`, this acquires one token from the + process-wide rate limiter so probe traffic counts against the + same aggregate per-provider budget as the CLI rows. `rate_limiter` + is a test seam; production callers should leave it unset. + """ + limiter = rate_limiter if rate_limiter is not None else get_default_limiter() + limiter.acquire(infer_provider(model)) + + url = base_url.rstrip("/") + "/v1/messages" + payload = { + "model": model, + "max_tokens": 64, + "messages": [ + { + "role": "user", + "content": ( + "If you have a tool to discover other tools, use it to " + "find one. Otherwise reply with the word 'done'." + ), + } + ], + "tools": [ + # The tool_search discovery tool itself. Type is the SDK- + # version-pinned `_20251119` suffix; name is the canonical + # `tool_search_tool_regex` (no suffix) Anthropic accepts. + # LiteLLM keys its beta-header translation on the type. + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex", + }, + # A trivial user tool for the discovery tool to potentially + # surface. Without at least one non-search tool the request + # is shape-valid but semantically empty; we include one so + # the wire shape mirrors what real Claude Code sends. + { + "name": "add_numbers", + "description": "Add two integers", + "input_schema": { + "type": "object", + "properties": { + "a": {"type": "integer"}, + "b": {"type": "integer"}, + }, + "required": ["a", "b"], + }, + }, + ], + } + try: + response = httpx.post( + url, + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "anthropic-version": "2023-06-01", + }, + json=payload, + timeout=timeout, + ) + except httpx.HTTPError as exc: + return ProbeResult(status_code=0, body="", error=f"transport: {exc}") + + body = response.text or "" + try: + payload_out = response.json() if body else None + except (json.JSONDecodeError, ValueError): + payload_out = None + + return ProbeResult( + status_code=response.status_code, + body=body, + payload=payload_out, + ) + + +def assert_tool_search_shape(result: ProbeResult) -> Optional[str]: + """Return None on success, else describe the first violation. + + Acceptance criteria: + + 1. HTTP status is 200 (no 400 from the upstream rejecting the + tool_search tool type or a missing beta header). + 2. Body is valid JSON. + 3. Body has either `content` (Anthropic-shape passthrough) or + `choices` (LiteLLM normalized openai-shape, used by Bedrock + Converse). Either is acceptable -- the matrix cares that the + proxy *accepts and forwards* tool_search, not that the model + actually chose to invoke it. Tool-invocation behavior is a + model decision the matrix has no business asserting on. + + The cell goes red when the upstream rejects the tool type, the + proxy drops the beta header, or the response shape is unusable. + Anything else (model decided to call or not call tool_search) is + irrelevant for this row. + """ + if result.error is not None: + return f"transport error: {result.error}" + if result.status_code != 200: + return f"status {result.status_code}: {result.body[:400]}" + if result.payload is None: + return f"non-JSON body: {result.body[:400]}" + if not isinstance(result.payload, Mapping): + return f"body is not a JSON object: {type(result.payload).__name__}" + # LiteLLM normalizes some provider responses to OpenAI shape + # (`choices`) and passes others through Anthropic-shape (`content`). + # Accept either; both prove the proxy round-tripped the request. + if "content" not in result.payload and "choices" not in result.payload: + return ( + f"response has neither `content` nor `choices`: " + f"keys={sorted(result.payload.keys())}" + ) + return None + + +def assert_count_tokens_shape(result: ProbeResult) -> Optional[str]: + """Return None on success, or an error string describing the first violation. + + Acceptance criteria are intentionally minimal: + + 1. HTTP status is 200. + 2. Body is valid JSON. + 3. Body has an `input_tokens` key whose value is a positive int. + + Anything beyond that (cache token fields, server metadata) is + optional and varies by provider/transport. Asserting on extras + would create a brittle test that flips red on neutral protocol + drift; matrix cells should only go red on functional regressions + a Claude Code user would feel. + """ + if result.error is not None: + return f"transport error: {result.error}" + if result.status_code != 200: + return f"status {result.status_code}: {result.body[:400]}" + if result.payload is None: + return f"non-JSON body: {result.body[:400]}" + if not isinstance(result.payload, Mapping): + return f"body is not a JSON object: {type(result.payload).__name__}" + tokens = result.payload.get("input_tokens") + if not isinstance(tokens, int) or isinstance(tokens, bool): + return f"input_tokens missing or not an int: got {tokens!r}" + if tokens <= 0: + return f"input_tokens must be positive; got {tokens}" + return None diff --git a/tests/e2e/claude_code/long_context_1m/__init__.py b/tests/e2e/claude_code/long_context_1m/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/long_context_1m/test_anthropic.py b/tests/e2e/claude_code/long_context_1m/test_anthropic.py new file mode 100644 index 00000000000..fb74d5fd40d --- /dev/null +++ b/tests/e2e/claude_code/long_context_1m/test_anthropic.py @@ -0,0 +1,224 @@ +"""long_context_1m x Anthropic. + +Drive the real `claude` CLI in headless mode with a ~210k-token padded +prompt and the `--betas context-1m-2025-08-07` beta header, route +through a LiteLLM proxy aimed at Anthropic, and assert the request +round-trips: no 400 from a stripped beta header, no 413 from a body +the proxy refused to forward, and a non-empty assistant reply at the +end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/long_context_1m/test_anthropic.py + ^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider + +Cost note (read this before scaling the prompt up): + +This row genuinely exercises the long-context path -- a 210k-token +prompt is *just over* Claude's standard 200k context window, which is +the threshold that requires the `context-1m-2025-08-07` beta header +to be honored end-to-end. Anything shorter would only test whether +the proxy forwards the beta header byte-for-byte; it would not catch +provider-side regressions where the header is forwarded but the +upstream silently truncates beyond the standard context (we've seen +this on third-party gateways). Anything longer is wasted spend. + +Per-cell cost at 210k input tokens: ~$0.63 Sonnet, ~$3.15 Opus. +Daily cost across all five providers (this row only): ~$19. + +Haiku 4.5 is intentionally omitted: it does not support 1M context +(its window is 200k). Reporting `not_applicable` for Haiku would +flip the entire cell to `not_applicable`, hiding genuine 1M +regressions on Sonnet/Opus; instead we exclude Haiku from the model +list entirely and let the matrix's per-cell aggregator green the +cell on Sonnet + Opus passing. This is the one row where the "all +three tiers must pass" rule is relaxed; it's relaxed structurally +(via the model list), not semantically (via not_applicable), so the +matrix builder stays unmodified. + +The prompt is delivered via subprocess stdin rather than a positional +argument. ARG_MAX on Linux is typically 2MB and an 840KB prompt +fits within that comfortably, but stdin is safer (no shell escaping +surprises, no surprise ARG_MAX clamp on a tightened sandbox) and +keeps the driver's `extra_args` slot free for the `--betas` flag. + +`--max-budget-usd 6` is a runaway-loop guard: a misbehaving test +that loops three Claude tiers in a single cell can't accidentally +spend more than ~$18 on this cell. The cap is twice the expected +worst case (Opus @ 210k = $3.15) plus a 50% margin. Tighten it if a +provider's pricing changes and the matrix starts spending more than +$10/day on this row. +""" + +from __future__ import annotations + +import os +from typing import Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +# Haiku 4.5 is excluded -- only Sonnet 4.6 and Opus 4.7 support the +# 1M-context beta. See module docstring for the per-cell-aggregator +# rationale. +ANTHROPIC_MODELS: Sequence[str] = ( + "claude-sonnet-4-6", + "claude-opus-4-7", +) + +# Beta header that opts an Anthropic-shape model into the 1M context +# window. Same string is accepted on Bedrock (Invoke + Converse) and +# Vertex per LiteLLM's transformers -- no per-provider translation is +# needed for this header, unlike `advanced-tool-use-2025-11-20` / +# `tool-search-tool-2025-10-19`. +LONG_CONTEXT_BETA = "context-1m-2025-08-07" + +# Target a padded prompt that lands just above Claude's standard 200k +# context window so the request can only succeed if the +# `context-1m-2025-08-07` beta header survives all the way to the +# upstream. Below 200k the cell would silently pass even with a +# proxy-dropped beta header; above ~220k we're paying for tokens that +# don't add signal. +TARGET_INPUT_TOKENS = 210_000 + +# Anthropic's English tokenizer averages ~4 chars/token. We cycle +# through several benign pangrams + filler so the padding looks like a +# real document, not a repeating monolith. Identical-line padding + +# "ignore everything above" trips Opus 4.7's safety filter as a +# suspected prompt-injection attempt -- we hit that during smoke +# testing and the cell flipped red for the wrong reason. Varied prose +# with a natural document-style framing keeps the filter quiet. +_PAD_CHUNKS = ( + "The quick brown fox jumps over the lazy dog. ", + "She sells seashells by the seashore on Sunday mornings. ", + "Pack my box with five dozen liquor jugs for the journey. ", + "How vexingly quick daft zebras jump over fences at dawn. ", + "Sphinx of black quartz, judge my vow of silence and patience. ", + "Waltz, bad nymph, for quick jigs in the moonlit meadow. ", + "Glib jocks quiz nymph to vex dwarf with a riddle of stone. ", + "Crazy Fredrick bought many very exquisite opal jewels lately. ", +) +_CHARS_PER_TOKEN = 4 + + +def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: + """Build a ~target_tokens-token padded prompt with a trailing instruction. + + Framing: + + - Lead with a benign document-style preamble that justifies the + long context (so safety filters see the prompt as "long + document review" rather than "adversarial padding"). + - Cycle through a small set of pangrams + filler sentences for + the bulk of the padding. Variety matters: identical repeated + lines look like a denial-of-service or injection attempt to + Anthropic's content filter on the larger tiers. + - End with the actual question. Claude's instruction-following + is stronger on recent tokens, so a 210k-token-into-the-past + instruction would risk a false-fail where the model ignores + it. + + `target_tokens` is an approximation: actual token count depends + on the tokenizer, but Anthropic's English tokenizer averages + ~4 chars/token, so 4 × target_tokens chars of padding gets us + close enough to the 1M-beta threshold (200k) that the proxy's + beta-header handling is the only path to success. + """ + preamble = ( + "I'm going to share an excerpt from a long document with you. " + "It contains a mix of practice sentences a typist might use to " + "warm up; treat the bulk of the text as background context. " + "I'll ask a short question at the end.\n\n" + "Begin excerpt:\n\n" + ) + closing = "\n\nEnd of excerpt. Please reply with the single word 'ok'." + + pad_target_chars = target_tokens * _CHARS_PER_TOKEN - len(preamble) - len(closing) + pad_lines = [] + pad_len = 0 + idx = 0 + while pad_len < pad_target_chars: + chunk = _PAD_CHUNKS[idx % len(_PAD_CHUNKS)] + pad_lines.append(chunk) + pad_len += len(chunk) + idx += 1 + return preamble + "".join(pad_lines) + closing + + +def test_long_context_1m_anthropic(compat_result): + """Drive the `claude` CLI with a ~210k-token prompt and the + `context-1m-2025-08-07` beta header; assert no 400 / 413 and a + non-empty reply for Sonnet + Opus.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + long_prompt = _build_long_prompt() + + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt=None, + stdin_input=long_prompt, + base_url=base_url, + api_key=api_key, + extra_args=[ + "--betas", + LONG_CONTEXT_BETA, + # Hard ceiling so a runaway test cannot blow the budget. + # See module docstring for sizing. + "--max-budget-usd", + "6", + ], + # Long-context requests can take a couple of minutes on a + # loaded upstream; the driver's default 120s is too tight. + timeout=300.0, + ) + + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/long_context_1m/test_azure.py b/tests/e2e/claude_code/long_context_1m/test_azure.py new file mode 100644 index 00000000000..5800fdadbfc --- /dev/null +++ b/tests/e2e/claude_code/long_context_1m/test_azure.py @@ -0,0 +1,224 @@ +"""long_context_1m x Azure (Microsoft Foundry). + +Drive the real `claude` CLI in headless mode with a ~210k-token padded +prompt and the `--betas context-1m-2025-08-07` beta header, route +through a LiteLLM proxy aimed at Anthropic, and assert the request +round-trips: no 400 from a stripped beta header, no 413 from a body +the proxy refused to forward, and a non-empty assistant reply at the +end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/long_context_1m/test_azure.py + ^^^^^^^^^^^^^^^ ^^^^^ + feature_id provider + +Cost note (read this before scaling the prompt up): + +This row genuinely exercises the long-context path -- a 210k-token +prompt is *just over* Claude's standard 200k context window, which is +the threshold that requires the `context-1m-2025-08-07` beta header +to be honored end-to-end. Anything shorter would only test whether +the proxy forwards the beta header byte-for-byte; it would not catch +provider-side regressions where the header is forwarded but the +upstream silently truncates beyond the standard context (we've seen +this on third-party gateways). Anything longer is wasted spend. + +Per-cell cost at 210k input tokens: ~$0.63 Sonnet, ~$3.15 Opus. +Daily cost across all five providers (this row only): ~$19. + +Haiku 4.5 is intentionally omitted: it does not support 1M context +(its window is 200k). Reporting `not_applicable` for Haiku would +flip the entire cell to `not_applicable`, hiding genuine 1M +regressions on Sonnet/Opus; instead we exclude Haiku from the model +list entirely and let the matrix's per-cell aggregator green the +cell on Sonnet + Opus passing. This is the one row where the "all +three tiers must pass" rule is relaxed; it's relaxed structurally +(via the model list), not semantically (via not_applicable), so the +matrix builder stays unmodified. + +The prompt is delivered via subprocess stdin rather than a positional +argument. ARG_MAX on Linux is typically 2MB and an 840KB prompt +fits within that comfortably, but stdin is safer (no shell escaping +surprises, no surprise ARG_MAX clamp on a tightened sandbox) and +keeps the driver's `extra_args` slot free for the `--betas` flag. + +`--max-budget-usd 6` is a runaway-loop guard: a misbehaving test +that loops three Claude tiers in a single cell can't accidentally +spend more than ~$18 on this cell. The cap is twice the expected +worst case (Opus @ 210k = $3.15) plus a 50% margin. Tighten it if a +provider's pricing changes and the matrix starts spending more than +$10/day on this row. +""" + +from __future__ import annotations + +import os +from typing import Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +# Haiku 4.5 is excluded -- only Sonnet 4.6 and Opus 4.7 support the +# 1M-context beta. See module docstring for the per-cell-aggregator +# rationale. +AZURE_MODELS: Sequence[str] = ( + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +) + +# Beta header that opts an Anthropic-shape model into the 1M context +# window. Same string is accepted on Bedrock (Invoke + Converse) and +# Vertex per LiteLLM's transformers -- no per-provider translation is +# needed for this header, unlike `advanced-tool-use-2025-11-20` / +# `tool-search-tool-2025-10-19`. +LONG_CONTEXT_BETA = "context-1m-2025-08-07" + +# Target a padded prompt that lands just above Claude's standard 200k +# context window so the request can only succeed if the +# `context-1m-2025-08-07` beta header survives all the way to the +# upstream. Below 200k the cell would silently pass even with a +# proxy-dropped beta header; above ~220k we're paying for tokens that +# don't add signal. +TARGET_INPUT_TOKENS = 210_000 + +# Anthropic's English tokenizer averages ~4 chars/token. We cycle +# through several benign pangrams + filler so the padding looks like a +# real document, not a repeating monolith. Identical-line padding + +# "ignore everything above" trips Opus 4.7's safety filter as a +# suspected prompt-injection attempt -- we hit that during smoke +# testing and the cell flipped red for the wrong reason. Varied prose +# with a natural document-style framing keeps the filter quiet. +_PAD_CHUNKS = ( + "The quick brown fox jumps over the lazy dog. ", + "She sells seashells by the seashore on Sunday mornings. ", + "Pack my box with five dozen liquor jugs for the journey. ", + "How vexingly quick daft zebras jump over fences at dawn. ", + "Sphinx of black quartz, judge my vow of silence and patience. ", + "Waltz, bad nymph, for quick jigs in the moonlit meadow. ", + "Glib jocks quiz nymph to vex dwarf with a riddle of stone. ", + "Crazy Fredrick bought many very exquisite opal jewels lately. ", +) +_CHARS_PER_TOKEN = 4 + + +def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: + """Build a ~target_tokens-token padded prompt with a trailing instruction. + + Framing: + + - Lead with a benign document-style preamble that justifies the + long context (so safety filters see the prompt as "long + document review" rather than "adversarial padding"). + - Cycle through a small set of pangrams + filler sentences for + the bulk of the padding. Variety matters: identical repeated + lines look like a denial-of-service or injection attempt to + Anthropic's content filter on the larger tiers. + - End with the actual question. Claude's instruction-following + is stronger on recent tokens, so a 210k-token-into-the-past + instruction would risk a false-fail where the model ignores + it. + + `target_tokens` is an approximation: actual token count depends + on the tokenizer, but Anthropic's English tokenizer averages + ~4 chars/token, so 4 × target_tokens chars of padding gets us + close enough to the 1M-beta threshold (200k) that the proxy's + beta-header handling is the only path to success. + """ + preamble = ( + "I'm going to share an excerpt from a long document with you. " + "It contains a mix of practice sentences a typist might use to " + "warm up; treat the bulk of the text as background context. " + "I'll ask a short question at the end.\n\n" + "Begin excerpt:\n\n" + ) + closing = "\n\nEnd of excerpt. Please reply with the single word 'ok'." + + pad_target_chars = target_tokens * _CHARS_PER_TOKEN - len(preamble) - len(closing) + pad_lines = [] + pad_len = 0 + idx = 0 + while pad_len < pad_target_chars: + chunk = _PAD_CHUNKS[idx % len(_PAD_CHUNKS)] + pad_lines.append(chunk) + pad_len += len(chunk) + idx += 1 + return preamble + "".join(pad_lines) + closing + + +def test_long_context_1m_azure(compat_result): + """Drive the `claude` CLI (Azure (Microsoft Foundry)) with a ~210k-token prompt and the + `context-1m-2025-08-07` beta header; assert no 400 / 413 and a + non-empty reply for Sonnet + Opus.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + long_prompt = _build_long_prompt() + + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt=None, + stdin_input=long_prompt, + base_url=base_url, + api_key=api_key, + extra_args=[ + "--betas", + LONG_CONTEXT_BETA, + # Hard ceiling so a runaway test cannot blow the budget. + # See module docstring for sizing. + "--max-budget-usd", + "6", + ], + # Long-context requests can take a couple of minutes on a + # loaded upstream; the driver's default 120s is too tight. + timeout=300.0, + ) + + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py b/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py new file mode 100644 index 00000000000..18587f7c2d6 --- /dev/null +++ b/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py @@ -0,0 +1,224 @@ +"""long_context_1m x Bedrock (Converse). + +Drive the real `claude` CLI in headless mode with a ~210k-token padded +prompt and the `--betas context-1m-2025-08-07` beta header, route +through a LiteLLM proxy aimed at Anthropic, and assert the request +round-trips: no 400 from a stripped beta header, no 413 from a body +the proxy refused to forward, and a non-empty assistant reply at the +end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py + ^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider + +Cost note (read this before scaling the prompt up): + +This row genuinely exercises the long-context path -- a 210k-token +prompt is *just over* Claude's standard 200k context window, which is +the threshold that requires the `context-1m-2025-08-07` beta header +to be honored end-to-end. Anything shorter would only test whether +the proxy forwards the beta header byte-for-byte; it would not catch +provider-side regressions where the header is forwarded but the +upstream silently truncates beyond the standard context (we've seen +this on third-party gateways). Anything longer is wasted spend. + +Per-cell cost at 210k input tokens: ~$0.63 Sonnet, ~$3.15 Opus. +Daily cost across all five providers (this row only): ~$19. + +Haiku 4.5 is intentionally omitted: it does not support 1M context +(its window is 200k). Reporting `not_applicable` for Haiku would +flip the entire cell to `not_applicable`, hiding genuine 1M +regressions on Sonnet/Opus; instead we exclude Haiku from the model +list entirely and let the matrix's per-cell aggregator green the +cell on Sonnet + Opus passing. This is the one row where the "all +three tiers must pass" rule is relaxed; it's relaxed structurally +(via the model list), not semantically (via not_applicable), so the +matrix builder stays unmodified. + +The prompt is delivered via subprocess stdin rather than a positional +argument. ARG_MAX on Linux is typically 2MB and an 840KB prompt +fits within that comfortably, but stdin is safer (no shell escaping +surprises, no surprise ARG_MAX clamp on a tightened sandbox) and +keeps the driver's `extra_args` slot free for the `--betas` flag. + +`--max-budget-usd 6` is a runaway-loop guard: a misbehaving test +that loops three Claude tiers in a single cell can't accidentally +spend more than ~$18 on this cell. The cap is twice the expected +worst case (Opus @ 210k = $3.15) plus a 50% margin. Tighten it if a +provider's pricing changes and the matrix starts spending more than +$10/day on this row. +""" + +from __future__ import annotations + +import os +from typing import Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +# Haiku 4.5 is excluded -- only Sonnet 4.6 and Opus 4.7 support the +# 1M-context beta. See module docstring for the per-cell-aggregator +# rationale. +BEDROCK_CONVERSE_MODELS: Sequence[str] = ( + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +) + +# Beta header that opts an Anthropic-shape model into the 1M context +# window. Same string is accepted on Bedrock (Invoke + Converse) and +# Vertex per LiteLLM's transformers -- no per-provider translation is +# needed for this header, unlike `advanced-tool-use-2025-11-20` / +# `tool-search-tool-2025-10-19`. +LONG_CONTEXT_BETA = "context-1m-2025-08-07" + +# Target a padded prompt that lands just above Claude's standard 200k +# context window so the request can only succeed if the +# `context-1m-2025-08-07` beta header survives all the way to the +# upstream. Below 200k the cell would silently pass even with a +# proxy-dropped beta header; above ~220k we're paying for tokens that +# don't add signal. +TARGET_INPUT_TOKENS = 210_000 + +# Anthropic's English tokenizer averages ~4 chars/token. We cycle +# through several benign pangrams + filler so the padding looks like a +# real document, not a repeating monolith. Identical-line padding + +# "ignore everything above" trips Opus 4.7's safety filter as a +# suspected prompt-injection attempt -- we hit that during smoke +# testing and the cell flipped red for the wrong reason. Varied prose +# with a natural document-style framing keeps the filter quiet. +_PAD_CHUNKS = ( + "The quick brown fox jumps over the lazy dog. ", + "She sells seashells by the seashore on Sunday mornings. ", + "Pack my box with five dozen liquor jugs for the journey. ", + "How vexingly quick daft zebras jump over fences at dawn. ", + "Sphinx of black quartz, judge my vow of silence and patience. ", + "Waltz, bad nymph, for quick jigs in the moonlit meadow. ", + "Glib jocks quiz nymph to vex dwarf with a riddle of stone. ", + "Crazy Fredrick bought many very exquisite opal jewels lately. ", +) +_CHARS_PER_TOKEN = 4 + + +def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: + """Build a ~target_tokens-token padded prompt with a trailing instruction. + + Framing: + + - Lead with a benign document-style preamble that justifies the + long context (so safety filters see the prompt as "long + document review" rather than "adversarial padding"). + - Cycle through a small set of pangrams + filler sentences for + the bulk of the padding. Variety matters: identical repeated + lines look like a denial-of-service or injection attempt to + Anthropic's content filter on the larger tiers. + - End with the actual question. Claude's instruction-following + is stronger on recent tokens, so a 210k-token-into-the-past + instruction would risk a false-fail where the model ignores + it. + + `target_tokens` is an approximation: actual token count depends + on the tokenizer, but Anthropic's English tokenizer averages + ~4 chars/token, so 4 × target_tokens chars of padding gets us + close enough to the 1M-beta threshold (200k) that the proxy's + beta-header handling is the only path to success. + """ + preamble = ( + "I'm going to share an excerpt from a long document with you. " + "It contains a mix of practice sentences a typist might use to " + "warm up; treat the bulk of the text as background context. " + "I'll ask a short question at the end.\n\n" + "Begin excerpt:\n\n" + ) + closing = "\n\nEnd of excerpt. Please reply with the single word 'ok'." + + pad_target_chars = target_tokens * _CHARS_PER_TOKEN - len(preamble) - len(closing) + pad_lines = [] + pad_len = 0 + idx = 0 + while pad_len < pad_target_chars: + chunk = _PAD_CHUNKS[idx % len(_PAD_CHUNKS)] + pad_lines.append(chunk) + pad_len += len(chunk) + idx += 1 + return preamble + "".join(pad_lines) + closing + + +def test_long_context_1m_bedrock_converse(compat_result): + """Drive the `claude` CLI (Bedrock (Converse)) with a ~210k-token prompt and the + `context-1m-2025-08-07` beta header; assert no 400 / 413 and a + non-empty reply for Sonnet + Opus.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + long_prompt = _build_long_prompt() + + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt=None, + stdin_input=long_prompt, + base_url=base_url, + api_key=api_key, + extra_args=[ + "--betas", + LONG_CONTEXT_BETA, + # Hard ceiling so a runaway test cannot blow the budget. + # See module docstring for sizing. + "--max-budget-usd", + "6", + ], + # Long-context requests can take a couple of minutes on a + # loaded upstream; the driver's default 120s is too tight. + timeout=300.0, + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py b/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py new file mode 100644 index 00000000000..0270197ce2a --- /dev/null +++ b/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py @@ -0,0 +1,224 @@ +"""long_context_1m x Bedrock (Invoke). + +Drive the real `claude` CLI in headless mode with a ~210k-token padded +prompt and the `--betas context-1m-2025-08-07` beta header, route +through a LiteLLM proxy aimed at Anthropic, and assert the request +round-trips: no 400 from a stripped beta header, no 413 from a body +the proxy refused to forward, and a non-empty assistant reply at the +end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py + ^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider + +Cost note (read this before scaling the prompt up): + +This row genuinely exercises the long-context path -- a 210k-token +prompt is *just over* Claude's standard 200k context window, which is +the threshold that requires the `context-1m-2025-08-07` beta header +to be honored end-to-end. Anything shorter would only test whether +the proxy forwards the beta header byte-for-byte; it would not catch +provider-side regressions where the header is forwarded but the +upstream silently truncates beyond the standard context (we've seen +this on third-party gateways). Anything longer is wasted spend. + +Per-cell cost at 210k input tokens: ~$0.63 Sonnet, ~$3.15 Opus. +Daily cost across all five providers (this row only): ~$19. + +Haiku 4.5 is intentionally omitted: it does not support 1M context +(its window is 200k). Reporting `not_applicable` for Haiku would +flip the entire cell to `not_applicable`, hiding genuine 1M +regressions on Sonnet/Opus; instead we exclude Haiku from the model +list entirely and let the matrix's per-cell aggregator green the +cell on Sonnet + Opus passing. This is the one row where the "all +three tiers must pass" rule is relaxed; it's relaxed structurally +(via the model list), not semantically (via not_applicable), so the +matrix builder stays unmodified. + +The prompt is delivered via subprocess stdin rather than a positional +argument. ARG_MAX on Linux is typically 2MB and an 840KB prompt +fits within that comfortably, but stdin is safer (no shell escaping +surprises, no surprise ARG_MAX clamp on a tightened sandbox) and +keeps the driver's `extra_args` slot free for the `--betas` flag. + +`--max-budget-usd 6` is a runaway-loop guard: a misbehaving test +that loops three Claude tiers in a single cell can't accidentally +spend more than ~$18 on this cell. The cap is twice the expected +worst case (Opus @ 210k = $3.15) plus a 50% margin. Tighten it if a +provider's pricing changes and the matrix starts spending more than +$10/day on this row. +""" + +from __future__ import annotations + +import os +from typing import Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +# Haiku 4.5 is excluded -- only Sonnet 4.6 and Opus 4.7 support the +# 1M-context beta. See module docstring for the per-cell-aggregator +# rationale. +BEDROCK_INVOKE_MODELS: Sequence[str] = ( + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +) + +# Beta header that opts an Anthropic-shape model into the 1M context +# window. Same string is accepted on Bedrock (Invoke + Converse) and +# Vertex per LiteLLM's transformers -- no per-provider translation is +# needed for this header, unlike `advanced-tool-use-2025-11-20` / +# `tool-search-tool-2025-10-19`. +LONG_CONTEXT_BETA = "context-1m-2025-08-07" + +# Target a padded prompt that lands just above Claude's standard 200k +# context window so the request can only succeed if the +# `context-1m-2025-08-07` beta header survives all the way to the +# upstream. Below 200k the cell would silently pass even with a +# proxy-dropped beta header; above ~220k we're paying for tokens that +# don't add signal. +TARGET_INPUT_TOKENS = 210_000 + +# Anthropic's English tokenizer averages ~4 chars/token. We cycle +# through several benign pangrams + filler so the padding looks like a +# real document, not a repeating monolith. Identical-line padding + +# "ignore everything above" trips Opus 4.7's safety filter as a +# suspected prompt-injection attempt -- we hit that during smoke +# testing and the cell flipped red for the wrong reason. Varied prose +# with a natural document-style framing keeps the filter quiet. +_PAD_CHUNKS = ( + "The quick brown fox jumps over the lazy dog. ", + "She sells seashells by the seashore on Sunday mornings. ", + "Pack my box with five dozen liquor jugs for the journey. ", + "How vexingly quick daft zebras jump over fences at dawn. ", + "Sphinx of black quartz, judge my vow of silence and patience. ", + "Waltz, bad nymph, for quick jigs in the moonlit meadow. ", + "Glib jocks quiz nymph to vex dwarf with a riddle of stone. ", + "Crazy Fredrick bought many very exquisite opal jewels lately. ", +) +_CHARS_PER_TOKEN = 4 + + +def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: + """Build a ~target_tokens-token padded prompt with a trailing instruction. + + Framing: + + - Lead with a benign document-style preamble that justifies the + long context (so safety filters see the prompt as "long + document review" rather than "adversarial padding"). + - Cycle through a small set of pangrams + filler sentences for + the bulk of the padding. Variety matters: identical repeated + lines look like a denial-of-service or injection attempt to + Anthropic's content filter on the larger tiers. + - End with the actual question. Claude's instruction-following + is stronger on recent tokens, so a 210k-token-into-the-past + instruction would risk a false-fail where the model ignores + it. + + `target_tokens` is an approximation: actual token count depends + on the tokenizer, but Anthropic's English tokenizer averages + ~4 chars/token, so 4 × target_tokens chars of padding gets us + close enough to the 1M-beta threshold (200k) that the proxy's + beta-header handling is the only path to success. + """ + preamble = ( + "I'm going to share an excerpt from a long document with you. " + "It contains a mix of practice sentences a typist might use to " + "warm up; treat the bulk of the text as background context. " + "I'll ask a short question at the end.\n\n" + "Begin excerpt:\n\n" + ) + closing = "\n\nEnd of excerpt. Please reply with the single word 'ok'." + + pad_target_chars = target_tokens * _CHARS_PER_TOKEN - len(preamble) - len(closing) + pad_lines = [] + pad_len = 0 + idx = 0 + while pad_len < pad_target_chars: + chunk = _PAD_CHUNKS[idx % len(_PAD_CHUNKS)] + pad_lines.append(chunk) + pad_len += len(chunk) + idx += 1 + return preamble + "".join(pad_lines) + closing + + +def test_long_context_1m_bedrock_invoke(compat_result): + """Drive the `claude` CLI (Bedrock (Invoke)) with a ~210k-token prompt and the + `context-1m-2025-08-07` beta header; assert no 400 / 413 and a + non-empty reply for Sonnet + Opus.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + long_prompt = _build_long_prompt() + + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt=None, + stdin_input=long_prompt, + base_url=base_url, + api_key=api_key, + extra_args=[ + "--betas", + LONG_CONTEXT_BETA, + # Hard ceiling so a runaway test cannot blow the budget. + # See module docstring for sizing. + "--max-budget-usd", + "6", + ], + # Long-context requests can take a couple of minutes on a + # loaded upstream; the driver's default 120s is too tight. + timeout=300.0, + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py b/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py new file mode 100644 index 00000000000..d2db4a1b4ee --- /dev/null +++ b/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py @@ -0,0 +1,224 @@ +"""long_context_1m x Vertex AI. + +Drive the real `claude` CLI in headless mode with a ~210k-token padded +prompt and the `--betas context-1m-2025-08-07` beta header, route +through a LiteLLM proxy aimed at Anthropic, and assert the request +round-trips: no 400 from a stripped beta header, no 413 from a body +the proxy refused to forward, and a non-empty assistant reply at the +end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/long_context_1m/test_vertex_ai.py + ^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider + +Cost note (read this before scaling the prompt up): + +This row genuinely exercises the long-context path -- a 210k-token +prompt is *just over* Claude's standard 200k context window, which is +the threshold that requires the `context-1m-2025-08-07` beta header +to be honored end-to-end. Anything shorter would only test whether +the proxy forwards the beta header byte-for-byte; it would not catch +provider-side regressions where the header is forwarded but the +upstream silently truncates beyond the standard context (we've seen +this on third-party gateways). Anything longer is wasted spend. + +Per-cell cost at 210k input tokens: ~$0.63 Sonnet, ~$3.15 Opus. +Daily cost across all five providers (this row only): ~$19. + +Haiku 4.5 is intentionally omitted: it does not support 1M context +(its window is 200k). Reporting `not_applicable` for Haiku would +flip the entire cell to `not_applicable`, hiding genuine 1M +regressions on Sonnet/Opus; instead we exclude Haiku from the model +list entirely and let the matrix's per-cell aggregator green the +cell on Sonnet + Opus passing. This is the one row where the "all +three tiers must pass" rule is relaxed; it's relaxed structurally +(via the model list), not semantically (via not_applicable), so the +matrix builder stays unmodified. + +The prompt is delivered via subprocess stdin rather than a positional +argument. ARG_MAX on Linux is typically 2MB and an 840KB prompt +fits within that comfortably, but stdin is safer (no shell escaping +surprises, no surprise ARG_MAX clamp on a tightened sandbox) and +keeps the driver's `extra_args` slot free for the `--betas` flag. + +`--max-budget-usd 6` is a runaway-loop guard: a misbehaving test +that loops three Claude tiers in a single cell can't accidentally +spend more than ~$18 on this cell. The cap is twice the expected +worst case (Opus @ 210k = $3.15) plus a 50% margin. Tighten it if a +provider's pricing changes and the matrix starts spending more than +$10/day on this row. +""" + +from __future__ import annotations + +import os +from typing import Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +# Haiku 4.5 is excluded -- only Sonnet 4.6 and Opus 4.7 support the +# 1M-context beta. See module docstring for the per-cell-aggregator +# rationale. +VERTEX_AI_MODELS: Sequence[str] = ( + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +) + +# Beta header that opts an Anthropic-shape model into the 1M context +# window. Same string is accepted on Bedrock (Invoke + Converse) and +# Vertex per LiteLLM's transformers -- no per-provider translation is +# needed for this header, unlike `advanced-tool-use-2025-11-20` / +# `tool-search-tool-2025-10-19`. +LONG_CONTEXT_BETA = "context-1m-2025-08-07" + +# Target a padded prompt that lands just above Claude's standard 200k +# context window so the request can only succeed if the +# `context-1m-2025-08-07` beta header survives all the way to the +# upstream. Below 200k the cell would silently pass even with a +# proxy-dropped beta header; above ~220k we're paying for tokens that +# don't add signal. +TARGET_INPUT_TOKENS = 210_000 + +# Anthropic's English tokenizer averages ~4 chars/token. We cycle +# through several benign pangrams + filler so the padding looks like a +# real document, not a repeating monolith. Identical-line padding + +# "ignore everything above" trips Opus 4.7's safety filter as a +# suspected prompt-injection attempt -- we hit that during smoke +# testing and the cell flipped red for the wrong reason. Varied prose +# with a natural document-style framing keeps the filter quiet. +_PAD_CHUNKS = ( + "The quick brown fox jumps over the lazy dog. ", + "She sells seashells by the seashore on Sunday mornings. ", + "Pack my box with five dozen liquor jugs for the journey. ", + "How vexingly quick daft zebras jump over fences at dawn. ", + "Sphinx of black quartz, judge my vow of silence and patience. ", + "Waltz, bad nymph, for quick jigs in the moonlit meadow. ", + "Glib jocks quiz nymph to vex dwarf with a riddle of stone. ", + "Crazy Fredrick bought many very exquisite opal jewels lately. ", +) +_CHARS_PER_TOKEN = 4 + + +def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: + """Build a ~target_tokens-token padded prompt with a trailing instruction. + + Framing: + + - Lead with a benign document-style preamble that justifies the + long context (so safety filters see the prompt as "long + document review" rather than "adversarial padding"). + - Cycle through a small set of pangrams + filler sentences for + the bulk of the padding. Variety matters: identical repeated + lines look like a denial-of-service or injection attempt to + Anthropic's content filter on the larger tiers. + - End with the actual question. Claude's instruction-following + is stronger on recent tokens, so a 210k-token-into-the-past + instruction would risk a false-fail where the model ignores + it. + + `target_tokens` is an approximation: actual token count depends + on the tokenizer, but Anthropic's English tokenizer averages + ~4 chars/token, so 4 × target_tokens chars of padding gets us + close enough to the 1M-beta threshold (200k) that the proxy's + beta-header handling is the only path to success. + """ + preamble = ( + "I'm going to share an excerpt from a long document with you. " + "It contains a mix of practice sentences a typist might use to " + "warm up; treat the bulk of the text as background context. " + "I'll ask a short question at the end.\n\n" + "Begin excerpt:\n\n" + ) + closing = "\n\nEnd of excerpt. Please reply with the single word 'ok'." + + pad_target_chars = target_tokens * _CHARS_PER_TOKEN - len(preamble) - len(closing) + pad_lines = [] + pad_len = 0 + idx = 0 + while pad_len < pad_target_chars: + chunk = _PAD_CHUNKS[idx % len(_PAD_CHUNKS)] + pad_lines.append(chunk) + pad_len += len(chunk) + idx += 1 + return preamble + "".join(pad_lines) + closing + + +def test_long_context_1m_vertex_ai(compat_result): + """Drive the `claude` CLI (Vertex AI) with a ~210k-token prompt and the + `context-1m-2025-08-07` beta header; assert no 400 / 413 and a + non-empty reply for Sonnet + Opus.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + long_prompt = _build_long_prompt() + + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt=None, + stdin_input=long_prompt, + base_url=base_url, + api_key=api_key, + extra_args=[ + "--betas", + LONG_CONTEXT_BETA, + # Hard ceiling so a runaway test cannot blow the budget. + # See module docstring for sizing. + "--max-budget-usd", + "6", + ], + # Long-context requests can take a couple of minutes on a + # loaded upstream; the driver's default 120s is too tight. + timeout=300.0, + ) + + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/manifest.yaml b/tests/e2e/claude_code/manifest.yaml new file mode 100644 index 00000000000..f7cccf0cef2 --- /dev/null +++ b/tests/e2e/claude_code/manifest.yaml @@ -0,0 +1,103 @@ +# Claude Code Compatibility Matrix — feature manifest. +# +# Defines the row order of the matrix and maps each feature_id to its +# human-readable display name. Adding a new feature to the matrix is a +# three-step change: +# 1. Append an entry to `features:` below. +# 2. Create a directory `tests/e2e/claude_code//`. +# 3. Add per-provider test files inside that directory. +# +# `feature_id` MUST match the directory name on disk; the test harness +# infers (feature, provider) for each test from its file path. + +schema_version: "1" + +# Provider column order in the rendered matrix. +providers: + - anthropic + - bedrock_invoke + - bedrock_converse + - vertex_ai + - azure + +# Feature row order. +features: + - id: basic_messaging_non_streaming + name: Basic messaging (non-streaming) + - id: basic_messaging_streaming + name: Basic messaging (streaming) + - id: tool_use + name: Tool use + - id: prompt_caching_5m + name: Prompt caching (5m TTL) + - id: vision + name: Vision + - id: thinking + name: Thinking + # The single row covers both API shapes Anthropic exposes — manual + # `thinking: {type: "enabled", budget_tokens: N}` (Haiku 4.5) and + # `thinking: {type: "adaptive"}` (Opus 4.7); Sonnet 4.6 supports + # either and Claude Code picks per model. A break in either + # transformer surfaces as a red cell because all three tiers must + # pass for the cell to go green. The row was named + # `extended_thinking` historically; Anthropic's docs now reserve + # that name for the deprecated manual mode only, so the row was + # renamed to the feature-level "Thinking". + - id: tool_use_streaming + name: Tool use (streaming / fine-grained) + - id: thinking_with_tool_use + name: Extended thinking + tool use + - id: pdf_input + name: PDF document input + - id: prompt_caching_1h + name: Prompt caching (1h TTL) + - id: web_search + name: Web search (server tool) + - id: structured_outputs + name: Structured outputs + # Drives `claude --json-schema ''`. Implementation note: + # Claude Code translates `--json-schema` to a synthetic + # `StructuredOutput` tool whose `input_schema` is the user's + # schema, then surfaces the tool_use input as + # `structured_output: {...}` on the trailing `result` event. + # This row tests that proxy-side handling of that tool round- + # trips end-to-end. It does NOT test Anthropic's server-side + # `output_config.schema` parameter (a separate feature used + # internally by Claude Code for session-title generation) -- + # `output_config` regressions surface in the HTTP-probe rows. + - id: count_tokens + name: count_tokens endpoint + # HTTP-probe row. Sends a direct POST to + # `{proxy}/v1/messages/count_tokens` for each Claude tier and + # asserts the response is shaped `{"input_tokens": }`. The CLI uses this endpoint internally but never + # surfaces its result in stream-json, so the only way to test + # the proxy's handling of it is to hit it directly. LiteLLM has + # shipped fixes here (e.g. Claude Code release-notes 2.1.121 + # "Vertex AI count_tokens returning 400 errors for proxy + # gateways"), which is exactly the regression class this row + # is meant to catch. + - id: tool_search + name: Tool search (MCP discovery) + # HTTP-probe row. Sends a request whose `tools` array includes + # a `tool_search_tool_regex_20251119` discovery tool and asserts + # the proxy + upstream accept it. This verifies LiteLLM's + # per-provider beta-header translation + # (`advanced-tool-use-2025-11-20` for Anthropic/Azure, + # `tool-search-tool-2025-10-19` for Vertex/Bedrock) is wired up. + # We deliberately don't try to trigger Claude Code's MCP-fan-out + # heuristic via `--mcp-config` -- that would couple the row to + # an internal behavior threshold that changes between Claude + # Code releases. The HTTP probe hits the bug surface LiteLLM + # has actually shipped fixes for (2.1.117, 2.1.72, 2.1.70 per + # the Claude Code release notes). + - id: long_context_1m + name: Long context (1M) + # Sends a ~210k-token padded prompt with the + # `context-1m-2025-08-07` beta header. Just-above the standard + # 200k context window so the request can only succeed when the + # beta header makes it all the way through the proxy to the + # upstream. Haiku 4.5 is intentionally omitted from this row's + # model list (its window is 200k); Sonnet 4.6 and Opus 4.7 are + # the only tiers exercised. Costs roughly $4/cell/run -- + # tighten the prompt-token target if pricing changes meaningfully. diff --git a/tests/e2e/claude_code/matrix_builder.py b/tests/e2e/claude_code/matrix_builder.py new file mode 100644 index 00000000000..5641e488da2 --- /dev/null +++ b/tests/e2e/claude_code/matrix_builder.py @@ -0,0 +1,198 @@ +"""Matrix JSON Builder. + +Pure-function module that consumes the pytest-produced `compat-results.json`, +the manifest, and run metadata, and emits the final `compatibility-matrix.json` +conforming to the schema published in the PRD. + +This module is deliberately free of subprocess, network, or filesystem side +effects in its public API — the public entry points take pre-loaded inputs +and return data structures, so they can be exercised by golden-file tests +without I/O. A small `build_from_paths()` convenience wrapper does the I/O +for callers that need it (the daily-cron publisher). +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence + +import yaml + +SCHEMA_VERSION = "1" +VALID_STATUSES = {"pass", "fail", "not_applicable", "not_tested"} + + +class ManifestError(ValueError): + """Raised when `manifest.yaml` is malformed.""" + + +class ResultsError(ValueError): + """Raised when the pytest results artifact is malformed.""" + + +def load_manifest(path: Path) -> Dict[str, Any]: + """Load and validate `manifest.yaml`. + + Returns a dict with keys: schema_version, providers, features. Raises + ManifestError on missing fields or schema mismatch. + """ + raw = yaml.safe_load(path.read_text()) + if not isinstance(raw, dict): + raise ManifestError(f"manifest at {path} is not a mapping") + schema_version = str(raw.get("schema_version", "")) + if schema_version != SCHEMA_VERSION: + raise ManifestError( + f"manifest schema_version {schema_version!r} does not match " + f"builder version {SCHEMA_VERSION!r}" + ) + providers = raw.get("providers") + if not isinstance(providers, list) or not providers: + raise ManifestError("manifest.providers must be a non-empty list") + features = raw.get("features") + if not isinstance(features, list) or not features: + raise ManifestError("manifest.features must be a non-empty list") + for feature in features: + if not isinstance(feature, dict): + raise ManifestError("each feature must be a mapping") + if not feature.get("id") or not feature.get("name"): + raise ManifestError("each feature must have id and name") + return raw + + +def load_results(path: Path) -> List[Dict[str, Any]]: + """Load the pytest results artifact and return its `results` list.""" + raw = json.loads(path.read_text()) + if not isinstance(raw, dict) or not isinstance(raw.get("results"), list): + raise ResultsError(f"results artifact at {path} has no `results` list") + return raw["results"] + + +def build_matrix( + *, + manifest: Mapping[str, Any], + results: Sequence[Mapping[str, Any]], + litellm_version: str, + claude_code_version: str, + generated_at: str, +) -> Dict[str, Any]: + """Build the published matrix JSON from pre-loaded inputs. + + Empty cells (no test ran for a (feature, provider) and no + `not_applicable` was declared) are filled in with `not_tested`. If + multiple results report on the same cell — e.g. a per-feature test + file containing one parametrize per Claude model — the cell aggregates + to `pass` only if every model passed; otherwise `fail` with the first + breaking model surfaced in the error. + """ + providers: List[str] = list(manifest["providers"]) + feature_specs: List[Dict[str, Any]] = list(manifest["features"]) + + grouped: Dict[tuple, List[Dict[str, Any]]] = {} + for entry in results: + if not isinstance(entry, Mapping): + continue + feature_id = entry.get("feature_id") + provider = entry.get("provider") + result = entry.get("result") + if not feature_id or not provider or not isinstance(result, Mapping): + continue + if result.get("status") not in VALID_STATUSES: + continue + grouped.setdefault((feature_id, provider), []).append(dict(result)) + + features_out: List[Dict[str, Any]] = [] + for spec in feature_specs: + feature_id = spec["id"] + cells: Dict[str, Dict[str, Any]] = {} + for provider in providers: + cell_results = grouped.get((feature_id, provider), []) + cells[provider] = _aggregate_cell(cell_results) + features_out.append( + { + "id": feature_id, + "name": spec["name"], + "providers": cells, + } + ) + + return { + "schema_version": SCHEMA_VERSION, + "generated_at": generated_at, + "litellm_version": litellm_version, + "claude_code_version": claude_code_version, + "providers": providers, + "features": features_out, + } + + +def _aggregate_cell(results: Sequence[Mapping[str, Any]]) -> Dict[str, Any]: + """Aggregate a list of per-model results into a single cell status. + + Order of precedence (most informative wins): + - Any `fail` → cell is `fail` with every failing model's error + joined by `"; "` so a multi-tier breakage doesn't silently hide + all but the first error from the published matrix. + - Any `pass` → cell is `pass`. A mix of (pass, not_applicable) — + e.g. a tier where the feature isn't supported alongside tiers + where it works — surfaces as `pass` so the published cell + reflects that the feature *does* work on this provider rather + than silently demoting it to `not_applicable` and discarding + the passing tiers. + - All `not_applicable` → cell is `not_applicable` with the first + row's reason. + - empty / nothing recognized → `not_tested`. + + `not_tested` rows are treated as absent data: they're dropped before + aggregation so a mix of (pass, not_tested) — e.g. from a partial + crash or a test that explicitly recorded "this tier didn't run" — + still surfaces the passing tiers rather than silently demoting the + whole cell to `not_tested`. A cell is only `not_tested` when *every* + row is `not_tested` (or there are no rows at all). + """ + if not results: + return {"status": "not_tested"} + + observed = [r for r in results if r.get("status") != "not_tested"] + if not observed: + return {"status": "not_tested"} + + failures = [r for r in observed if r.get("status") == "fail"] + if failures: + errors = [str(r.get("error", "test failed")) for r in failures] + return {"status": "fail", "error": "; ".join(errors)} + + if any(r.get("status") == "pass" for r in observed): + return {"status": "pass"} + + if all(r.get("status") == "not_applicable" for r in observed): + return { + "status": "not_applicable", + "reason": str(observed[0].get("reason", "not applicable")), + } + + return {"status": "not_tested"} + + +def build_from_paths( + *, + manifest_path: Path, + results_path: Path, + litellm_version: str, + claude_code_version: str, + generated_at: str, + output_path: Optional[Path] = None, +) -> Dict[str, Any]: + """I/O wrapper around build_matrix used by the publisher script.""" + manifest = load_manifest(manifest_path) + results = load_results(results_path) + matrix = build_matrix( + manifest=manifest, + results=results, + litellm_version=litellm_version, + claude_code_version=claude_code_version, + generated_at=generated_at, + ) + if output_path is not None: + output_path.write_text(json.dumps(matrix, indent=2, sort_keys=False) + "\n") + return matrix diff --git a/tests/e2e/claude_code/pdf_input/__init__.py b/tests/e2e/claude_code/pdf_input/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/pdf_input/test_anthropic.py b/tests/e2e/claude_code/pdf_input/test_anthropic.py new file mode 100644 index 00000000000..36fb69a1db6 --- /dev/null +++ b/tests/e2e/claude_code/pdf_input/test_anthropic.py @@ -0,0 +1,176 @@ +"""pdf_input x Anthropic. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Anthropic, write a tiny valid PDF to disk, allow the built-in `Read` +tool, and ask Claude to read the PDF and report what it contains. + +The Read tool inlines the PDF bytes as `document` content blocks on the +next assistant turn, which is exactly the gateway path we want to +exercise: it's distinct from image content blocks (which are tested in +`vision/`) and uses a different transformation in LiteLLM's Anthropic +provider. We assert the upstream produces a non-empty reply that +references the contents of the PDF — proving the proxy preserved the +document content block end-to-end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/pdf_input/test_anthropic.py + ^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +# Smallest valid PDF that renders a single visible word ("PONG"). Built +# inline rather than checked in as a binary fixture so the test stays +# self-contained and the marker word is easy to grep for in CI logs. +# The structure is a hand-crafted single-page PDF with one Helvetica +# text show; offsets are computed at write time so the xref table +# stays consistent regardless of platform line endings. +PDF_MARKER = "PONG" + + +def _build_minimal_pdf(marker: str) -> bytes: + """Return a single-page PDF whose only visible text is `marker`. + + We construct the PDF imperatively because `pypdf`/`reportlab` are + not in the test deps and we want the cell to work in a clean + environment. The xref offsets are recomputed for each `marker` + length so the file stays well-formed. + """ + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + ( + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] " + b"/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>" + ), + # Page content stream: position the text and show the marker. + # `BT ... ET` is a text object; `Tf` selects font, `Td` moves + # the cursor, `Tj` paints a string. + ( + b"<< /Length %d >>\nstream\nBT /F1 24 Tf 50 100 Td (" + + marker.encode("ascii") + + b") Tj ET\nendstream" + ), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ] + # Fix up the /Length on the content stream to match its body. + body_open = objects[3].index(b"\nstream\n") + len(b"\nstream\n") + body_close = objects[3].index(b"\nendstream") + body_len = body_close - body_open + objects[3] = ( + b"<< /Length " + + str(body_len).encode("ascii") + + b" >>\nstream\n" + + objects[3][body_open:body_close] + + b"\nendstream" + ) + + out = bytearray(b"%PDF-1.4\n") + offsets = [] + for i, obj in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n" + + xref_offset = len(out) + out += b"xref\n0 %d\n" % (len(objects) + 1) + out += b"0000000000 65535 f \n" + for off in offsets: + out += f"{off:010d} 00000 n \n".encode("ascii") + out += ( + b"trailer\n<< /Size " + + str(len(objects) + 1).encode("ascii") + + b" /Root 1 0 R >>\nstartxref\n" + + str(xref_offset).encode("ascii") + + b"\n%%EOF\n" + ) + return bytes(out) + + +def test_pdf_input_anthropic(compat_result, tmp_path): + """Drive the `claude` CLI against the LiteLLM proxy with a PDF + attached via the Read tool and assert the reply references it.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + pdf_path = tmp_path / "marker.pdf" + pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER)) + + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt=( + f"Use the Read tool to read the file at {pdf_path}. " + "Report the single word that appears in the document." + ), + base_url=base_url, + api_key=api_key, + extra_args=["--allowed-tools", "Read"], + ) + + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + # The strongest gateway-level signal we can assert without + # parsing every event type: the model's final user-visible + # reply names the marker word that only the PDF carries. + # If the proxy dropped the `document` content block, the + # model has no way to produce this token. + if PDF_MARKER not in outcome.text.upper(): + error = ( + f"[{model}] reply did not reference the PDF marker {PDF_MARKER!r}; " + f"got: {outcome.text.strip()!r}" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/pdf_input/test_azure.py b/tests/e2e/claude_code/pdf_input/test_azure.py new file mode 100644 index 00000000000..810c857e407 --- /dev/null +++ b/tests/e2e/claude_code/pdf_input/test_azure.py @@ -0,0 +1,146 @@ +"""pdf_input x Microsoft Foundry (Azure). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Microsoft Foundry's Anthropic deployments on Azure, +write a tiny valid PDF to disk, allow the built-in `Read` tool, and +ask Claude to read the PDF and report what it contains. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/pdf_input/test_azure.py + ^^^^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + +PDF_MARKER = "PONG" + + +def _build_minimal_pdf(marker: str) -> bytes: + """Return a single-page PDF whose only visible text is `marker`.""" + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + ( + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] " + b"/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>" + ), + ( + b"<< /Length %d >>\nstream\nBT /F1 24 Tf 50 100 Td (" + + marker.encode("ascii") + + b") Tj ET\nendstream" + ), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ] + body_open = objects[3].index(b"\nstream\n") + len(b"\nstream\n") + body_close = objects[3].index(b"\nendstream") + body_len = body_close - body_open + objects[3] = ( + b"<< /Length " + + str(body_len).encode("ascii") + + b" >>\nstream\n" + + objects[3][body_open:body_close] + + b"\nendstream" + ) + + out = bytearray(b"%PDF-1.4\n") + offsets = [] + for i, obj in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n" + + xref_offset = len(out) + out += b"xref\n0 %d\n" % (len(objects) + 1) + out += b"0000000000 65535 f \n" + for off in offsets: + out += f"{off:010d} 00000 n \n".encode("ascii") + out += ( + b"trailer\n<< /Size " + + str(len(objects) + 1).encode("ascii") + + b" /Root 1 0 R >>\nstartxref\n" + + str(xref_offset).encode("ascii") + + b"\n%%EOF\n" + ) + return bytes(out) + + +def test_pdf_input_azure(compat_result, tmp_path): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + pdf_path = tmp_path / "marker.pdf" + pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER)) + + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt=( + f"Use the Read tool to read the file at {pdf_path}. " + "Report the single word that appears in the document." + ), + base_url=base_url, + api_key=api_key, + extra_args=["--allowed-tools", "Read"], + ) + + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if PDF_MARKER not in outcome.text.upper(): + error = ( + f"[{model}] reply did not reference the PDF marker {PDF_MARKER!r}; " + f"got: {outcome.text.strip()!r}" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py b/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py new file mode 100644 index 00000000000..191a27c6d46 --- /dev/null +++ b/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py @@ -0,0 +1,152 @@ +"""pdf_input x Bedrock (Converse). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the `Converse` API, write a tiny +valid PDF to disk, allow the built-in `Read` tool, and ask Claude to +read the PDF and report what it contains. + +Bedrock Converse expresses documents via its own +`document = { format, name, source: { bytes } }` shape; this cell +catches gateway regressions where the proxy fails to translate +Anthropic's `document` content block to Converse's document format +(or vice versa on the response). + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/pdf_input/test_bedrock_converse.py + ^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + +PDF_MARKER = "PONG" + + +def _build_minimal_pdf(marker: str) -> bytes: + """Return a single-page PDF whose only visible text is `marker`.""" + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + ( + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] " + b"/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>" + ), + ( + b"<< /Length %d >>\nstream\nBT /F1 24 Tf 50 100 Td (" + + marker.encode("ascii") + + b") Tj ET\nendstream" + ), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ] + body_open = objects[3].index(b"\nstream\n") + len(b"\nstream\n") + body_close = objects[3].index(b"\nendstream") + body_len = body_close - body_open + objects[3] = ( + b"<< /Length " + + str(body_len).encode("ascii") + + b" >>\nstream\n" + + objects[3][body_open:body_close] + + b"\nendstream" + ) + + out = bytearray(b"%PDF-1.4\n") + offsets = [] + for i, obj in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n" + + xref_offset = len(out) + out += b"xref\n0 %d\n" % (len(objects) + 1) + out += b"0000000000 65535 f \n" + for off in offsets: + out += f"{off:010d} 00000 n \n".encode("ascii") + out += ( + b"trailer\n<< /Size " + + str(len(objects) + 1).encode("ascii") + + b" /Root 1 0 R >>\nstartxref\n" + + str(xref_offset).encode("ascii") + + b"\n%%EOF\n" + ) + return bytes(out) + + +def test_pdf_input_bedrock_converse(compat_result, tmp_path): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + pdf_path = tmp_path / "marker.pdf" + pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER)) + + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt=( + f"Use the Read tool to read the file at {pdf_path}. " + "Report the single word that appears in the document." + ), + base_url=base_url, + api_key=api_key, + extra_args=["--allowed-tools", "Read"], + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if PDF_MARKER not in outcome.text.upper(): + error = ( + f"[{model}] reply did not reference the PDF marker {PDF_MARKER!r}; " + f"got: {outcome.text.strip()!r}" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py b/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py new file mode 100644 index 00000000000..163cabb45a0 --- /dev/null +++ b/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py @@ -0,0 +1,151 @@ +"""pdf_input x Bedrock (Invoke). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the legacy `InvokeModel` API path, +write a tiny valid PDF to disk, allow the built-in `Read` tool, and +ask Claude to read the PDF and report what it contains. + +Bedrock InvokeModel for Anthropic models accepts the native +`document` content block shape; this cell catches gateway regressions +where the proxy drops or mis-encodes the document content block on +the way through (e.g. base64-only encoding, missing media_type, etc.). + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py + ^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + +PDF_MARKER = "PONG" + + +def _build_minimal_pdf(marker: str) -> bytes: + """Return a single-page PDF whose only visible text is `marker`.""" + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + ( + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] " + b"/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>" + ), + ( + b"<< /Length %d >>\nstream\nBT /F1 24 Tf 50 100 Td (" + + marker.encode("ascii") + + b") Tj ET\nendstream" + ), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ] + body_open = objects[3].index(b"\nstream\n") + len(b"\nstream\n") + body_close = objects[3].index(b"\nendstream") + body_len = body_close - body_open + objects[3] = ( + b"<< /Length " + + str(body_len).encode("ascii") + + b" >>\nstream\n" + + objects[3][body_open:body_close] + + b"\nendstream" + ) + + out = bytearray(b"%PDF-1.4\n") + offsets = [] + for i, obj in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n" + + xref_offset = len(out) + out += b"xref\n0 %d\n" % (len(objects) + 1) + out += b"0000000000 65535 f \n" + for off in offsets: + out += f"{off:010d} 00000 n \n".encode("ascii") + out += ( + b"trailer\n<< /Size " + + str(len(objects) + 1).encode("ascii") + + b" /Root 1 0 R >>\nstartxref\n" + + str(xref_offset).encode("ascii") + + b"\n%%EOF\n" + ) + return bytes(out) + + +def test_pdf_input_bedrock_invoke(compat_result, tmp_path): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + pdf_path = tmp_path / "marker.pdf" + pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER)) + + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt=( + f"Use the Read tool to read the file at {pdf_path}. " + "Report the single word that appears in the document." + ), + base_url=base_url, + api_key=api_key, + extra_args=["--allowed-tools", "Read"], + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if PDF_MARKER not in outcome.text.upper(): + error = ( + f"[{model}] reply did not reference the PDF marker {PDF_MARKER!r}; " + f"got: {outcome.text.strip()!r}" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/pdf_input/test_vertex_ai.py b/tests/e2e/claude_code/pdf_input/test_vertex_ai.py new file mode 100644 index 00000000000..0d0573d05b3 --- /dev/null +++ b/tests/e2e/claude_code/pdf_input/test_vertex_ai.py @@ -0,0 +1,146 @@ +"""pdf_input x Vertex AI. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to GCP Vertex AI, write a tiny valid PDF to disk, allow +the built-in `Read` tool, and ask Claude to read the PDF and report +what it contains. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/pdf_input/test_vertex_ai.py + ^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + +PDF_MARKER = "PONG" + + +def _build_minimal_pdf(marker: str) -> bytes: + """Return a single-page PDF whose only visible text is `marker`.""" + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + ( + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] " + b"/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>" + ), + ( + b"<< /Length %d >>\nstream\nBT /F1 24 Tf 50 100 Td (" + + marker.encode("ascii") + + b") Tj ET\nendstream" + ), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ] + body_open = objects[3].index(b"\nstream\n") + len(b"\nstream\n") + body_close = objects[3].index(b"\nendstream") + body_len = body_close - body_open + objects[3] = ( + b"<< /Length " + + str(body_len).encode("ascii") + + b" >>\nstream\n" + + objects[3][body_open:body_close] + + b"\nendstream" + ) + + out = bytearray(b"%PDF-1.4\n") + offsets = [] + for i, obj in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n" + + xref_offset = len(out) + out += b"xref\n0 %d\n" % (len(objects) + 1) + out += b"0000000000 65535 f \n" + for off in offsets: + out += f"{off:010d} 00000 n \n".encode("ascii") + out += ( + b"trailer\n<< /Size " + + str(len(objects) + 1).encode("ascii") + + b" /Root 1 0 R >>\nstartxref\n" + + str(xref_offset).encode("ascii") + + b"\n%%EOF\n" + ) + return bytes(out) + + +def test_pdf_input_vertex_ai(compat_result, tmp_path): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + pdf_path = tmp_path / "marker.pdf" + pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER)) + + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt=( + f"Use the Read tool to read the file at {pdf_path}. " + "Report the single word that appears in the document." + ), + base_url=base_url, + api_key=api_key, + extra_args=["--allowed-tools", "Read"], + ) + + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if PDF_MARKER not in outcome.text.upper(): + error = ( + f"[{model}] reply did not reference the PDF marker {PDF_MARKER!r}; " + f"got: {outcome.text.strip()!r}" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/pr_gate_version_resolver.py b/tests/e2e/claude_code/pr_gate_version_resolver.py new file mode 100644 index 00000000000..82e12a2bf15 --- /dev/null +++ b/tests/e2e/claude_code/pr_gate_version_resolver.py @@ -0,0 +1,150 @@ +"""Claude Code PR-Gate Version Resolver. + +Resolves the `@anthropic-ai/claude-code` npm version that the PR-gate CI +job installs. Selects the newest version (by publish timestamp) whose +publish timestamp is at least 3 days old. The 3-day window is a security +review buffer — see PRD #26476, "Version resolvers". + +Two surfaces: + +- ``resolve_pr_gate_version(...)`` — the importable function. Accepts + pre-fetched npm metadata (for unit tests) or a custom ``fetcher`` + callable. The default fetcher hits the public npm registry. +- ``python -m claude_code.pr_gate_version_resolver`` — prints the + resolved version string to stdout, suitable for piping into a shell + ``$(...)`` substitution inside the CircleCI job. + +The CLI form is what CircleCI runs at job start; engineers reading the +job log can see the selected version on a single line above the +``npm install -g`` step (acceptance criterion: "the selected Claude +Code version is logged in the CI output"). +""" + +from __future__ import annotations + +import json +import sys +import urllib.request +from datetime import datetime, timedelta, timezone +from typing import Callable, Mapping, Optional + +PACKAGE_NAME = "@anthropic-ai/claude-code" +NPM_REGISTRY_URL = "https://registry.npmjs.org/{package}" +DEFAULT_MIN_AGE = timedelta(days=3) +DEFAULT_FETCH_TIMEOUT_SECONDS = 30 + +# npm's `time` map mixes per-version timestamps with these meta keys. +_TIME_META_KEYS = frozenset({"created", "modified"}) + + +class NoEligibleVersionError(RuntimeError): + """Raised when no version in the npm metadata satisfies the min-age cutoff.""" + + +def _parse_npm_timestamp(value: str) -> datetime: + """Parse the ISO-8601 timestamps npm emits (always UTC, may use ``Z``).""" + if value.endswith("Z"): + value = value[:-1] + "+00:00" + parsed = datetime.fromisoformat(value) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed + + +def _default_fetcher(package_name: str) -> dict: + """Fetch the npm packument for ``package_name`` over HTTPS. + + Uses urllib (stdlib) so this module has no extra dependencies in the + CI environment. Returns the raw JSON dict. + """ + # urllib.parse.quote would encode the leading '@' / '/' which the + # npm registry expects literally; do a minimal hand-roll instead. + url = NPM_REGISTRY_URL.format(package=package_name.replace("/", "%2F")) + req = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen( # noqa: S310 — registry URL is constant + req, timeout=DEFAULT_FETCH_TIMEOUT_SECONDS + ) as response: + body = response.read().decode("utf-8") + return json.loads(body) + + +def resolve_pr_gate_version( + *, + metadata: Optional[Mapping] = None, + fetcher: Optional[Callable[[str], Mapping]] = None, + as_of: Optional[datetime] = None, + min_age: timedelta = DEFAULT_MIN_AGE, + package_name: str = PACKAGE_NAME, +) -> str: + """Return the newest npm version of ``package_name`` published >= ``min_age`` ago. + + "Newest" means newest by **publish time**, not semver string order — + if a patch lands on an older major after a newer release, the + patched line is the eligible one. + + Args: + metadata: Pre-fetched npm packument (skips the HTTP call). Useful + for unit tests. + fetcher: Callable taking a package name and returning the + packument. Defaults to a stdlib HTTPS fetcher. + as_of: The clock used to decide whether a version is "old + enough". Defaults to ``datetime.now(timezone.utc)``. + min_age: Minimum publish age. Defaults to 3 days. + package_name: Defaults to ``@anthropic-ai/claude-code``. + + Raises: + NoEligibleVersionError: when no version in the registry meets + the age cutoff. + """ + if metadata is None: + fetch = fetcher or _default_fetcher + metadata = fetch(package_name) + + times = metadata.get("time") or {} + if as_of is None: + as_of = datetime.now(timezone.utc) + cutoff = as_of - min_age + + eligible: list[tuple[datetime, str]] = [] + for version, raw_ts in times.items(): + if version in _TIME_META_KEYS: + continue + if not isinstance(raw_ts, str): + continue + if "-" in version: + continue + published = _parse_npm_timestamp(raw_ts) + if published <= cutoff: + eligible.append((published, version)) + + if not eligible: + raise NoEligibleVersionError( + f"no version of {package_name} is at least {min_age} old " + f"as of {as_of.isoformat()}" + ) + + eligible.sort(key=lambda pair: pair[0], reverse=True) + return eligible[0][1] + + +def _main(argv: list[str]) -> int: + """Print the resolved version to stdout. Exit code 0 on success. + + Stderr carries the human-readable announcement so the version can be + captured cleanly with ``$(python -m ...)`` in shell. + """ + try: + version = resolve_pr_gate_version() + except Exception as exc: # noqa: BLE001 — CLI surface, want everything + print(f"pr_gate_version_resolver: {exc}", file=sys.stderr) # noqa: T201 + return 1 + print( # noqa: T201 + f"pr_gate_version_resolver: selected {PACKAGE_NAME}@{version}", + file=sys.stderr, + ) + print(version) # noqa: T201 + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main(sys.argv[1:])) diff --git a/tests/e2e/claude_code/prompt_caching_1h/__init__.py b/tests/e2e/claude_code/prompt_caching_1h/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py b/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py new file mode 100644 index 00000000000..d81887231d8 --- /dev/null +++ b/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py @@ -0,0 +1,124 @@ +"""prompt_caching_1h x Anthropic. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Anthropic, opt into the 1-hour cache TTL via `ENABLE_PROMPT_CACHING_1H`, +and assert that the upstream's usage block reports either +`cache_creation_input_tokens` or `cache_read_input_tokens` > 0 — i.e. +the proxy preserved Claude Code's `cache_control: { ttl: "1h" }` +annotations end-to-end and the upstream actually honored them. + +This is the 1-hour-TTL companion to `prompt_caching_5m/`. It exists as +its own cell because the 1h TTL travels through the proxy with a +distinct `cache_control` shape (and a distinct beta-header gate on +some providers); a regression that strips or downgrades the TTL on the +way through is invisible to the 5m cell, which would still see cache +hits with a default-TTL annotation. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +# Per the changelog (2.1.108): `ENABLE_PROMPT_CACHING_1H` flips Claude +# Code from the default 5-minute cache TTL to a 1-hour TTL on the +# `cache_control` annotations it adds to the system prompt and the +# most recent user turn. Setting it here is what we are validating +# the proxy faithfully forwards to the upstream. +CACHE_1H_ENV = {"ENABLE_PROMPT_CACHING_1H": "1"} + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + """Return cache_creation_input_tokens + cache_read_input_tokens from + the upstream usage block, or 0 if the keys are missing.""" + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +def test_prompt_caching_1h_anthropic(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy with the 1h + TTL opt-in env var set, and assert the upstream usage block + surfaces a non-zero cache token count.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + extra_env=CACHE_1H_ENV, + ) + + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens with " + "ENABLE_PROMPT_CACHING_1H=1; the proxy likely stripped the " + "1h TTL beta header or rejected the cache_control shape" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_azure.py b/tests/e2e/claude_code/prompt_caching_1h/test_azure.py new file mode 100644 index 00000000000..416757f8691 --- /dev/null +++ b/tests/e2e/claude_code/prompt_caching_1h/test_azure.py @@ -0,0 +1,104 @@ +"""prompt_caching_1h x Microsoft Foundry (Azure). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Microsoft Foundry's Anthropic deployments on Azure, +opt into the 1-hour cache TTL via `ENABLE_PROMPT_CACHING_1H`, and +assert the upstream's usage block reports a non-zero cache token count. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/prompt_caching_1h/test_azure.py + ^^^^^^^^^^^^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + +CACHE_1H_ENV = {"ENABLE_PROMPT_CACHING_1H": "1"} + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +def test_prompt_caching_1h_azure(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + extra_env=CACHE_1H_ENV, + ) + + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens with " + "ENABLE_PROMPT_CACHING_1H=1" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py new file mode 100644 index 00000000000..5bc632c6f1b --- /dev/null +++ b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py @@ -0,0 +1,112 @@ +"""prompt_caching_1h x Bedrock (Converse). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the `Converse` API, opt into the +1-hour cache TTL via `ENABLE_PROMPT_CACHING_1H`, and assert the +upstream's usage block reports a non-zero cache token count. + +Bedrock Converse expresses prompt caching via `cachePoint` markers in +the message list, with TTL controlled out-of-band; this cell catches +proxy regressions where the 1h opt-in fails to translate into the +correct Converse cache configuration. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + +CACHE_1H_ENV = { + "ENABLE_PROMPT_CACHING_1H": "1", + "ENABLE_PROMPT_CACHING_1H_BEDROCK": "1", +} + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +def test_prompt_caching_1h_bedrock_converse(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + extra_env=CACHE_1H_ENV, + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens with " + "1h-TTL opt-in env vars set" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py new file mode 100644 index 00000000000..4501834956b --- /dev/null +++ b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py @@ -0,0 +1,117 @@ +"""prompt_caching_1h x Bedrock (Invoke). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the legacy `InvokeModel` API path, +opt into the 1-hour cache TTL via `ENABLE_PROMPT_CACHING_1H`, and +assert the upstream's usage block reports a non-zero cache token count. + +Bedrock historically gated 1h prompt caching behind a separate +`ENABLE_PROMPT_CACHING_1H_BEDROCK` env var (see 2.1.108: deprecated but +still honored). The proxy must accept either env var and forward an +appropriate `cache_control` shape to the Bedrock InvokeModel endpoint; +this cell catches regressions where the TTL is silently downgraded to +5 minutes on the way through. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + +# Set both the modern and the deprecated-but-honored Bedrock var so we +# match whichever code path the proxy is following. +CACHE_1H_ENV = { + "ENABLE_PROMPT_CACHING_1H": "1", + "ENABLE_PROMPT_CACHING_1H_BEDROCK": "1", +} + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +def test_prompt_caching_1h_bedrock_invoke(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + extra_env=CACHE_1H_ENV, + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens with " + "1h-TTL opt-in env vars set; the proxy likely stripped or " + "downgraded the cache_control TTL" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py b/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py new file mode 100644 index 00000000000..09ded634b45 --- /dev/null +++ b/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py @@ -0,0 +1,104 @@ +"""prompt_caching_1h x Vertex AI. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to GCP Vertex AI, opt into the 1-hour cache TTL via +`ENABLE_PROMPT_CACHING_1H`, and assert the upstream's usage block +reports a non-zero cache token count. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + +CACHE_1H_ENV = {"ENABLE_PROMPT_CACHING_1H": "1"} + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +def test_prompt_caching_1h_vertex_ai(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + extra_env=CACHE_1H_ENV, + ) + + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens with " + "ENABLE_PROMPT_CACHING_1H=1" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/prompt_caching_5m/__init__.py b/tests/e2e/claude_code/prompt_caching_5m/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py b/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py new file mode 100644 index 00000000000..4b20a65f31b --- /dev/null +++ b/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py @@ -0,0 +1,113 @@ +"""prompt_caching_5m x Anthropic. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Anthropic, and assert that the upstream's usage block reports either +`cache_creation_input_tokens` or `cache_read_input_tokens` > 0 — i.e. +the proxy preserves Claude Code's `cache_control` annotations end-to-end +and the upstream actually honored them. This is the 5-minute (default) +cache TTL row. + +Claude Code itself sets `cache_control: { type: "ephemeral" }` on the +system prompt and the most recent user turn for every request, so a +single live invocation is enough to surface a cache-creation count on +the first call and a cache-read count on a warm follow-up call. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + """Return cache_creation_input_tokens + cache_read_input_tokens from + the upstream usage block, or 0 if the keys are missing.""" + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +def test_prompt_caching_5m_anthropic(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert the + upstream usage block surfaces a non-zero cache token count.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + ) + + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens; " + "expected cache_control on the system prompt to produce a non-zero " + "cache_creation_input_tokens or cache_read_input_tokens" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_azure.py b/tests/e2e/claude_code/prompt_caching_5m/test_azure.py new file mode 100644 index 00000000000..22bd5aa7048 --- /dev/null +++ b/tests/e2e/claude_code/prompt_caching_5m/test_azure.py @@ -0,0 +1,111 @@ +"""prompt_caching_5m x Azure (Microsoft Foundry). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Anthropic's models hosted in Microsoft Foundry on +Azure, and assert that the upstream's usage block reports either +`cache_creation_input_tokens` or `cache_read_input_tokens` > 0. + +Foundry's Anthropic deployments honor the default 5-minute ephemeral +`cache_control` exactly like anthropic.com. The 1-hour `scope: "global"` +variant is *not* supported on Foundry — LiteLLM strips that field +before forwarding (see `_remove_scope_from_cache_control` in +`litellm/llms/azure_ai/anthropic/messages_transformation.py`) — but +this row exercises the 5-minute TTL only, so that quirk does not apply. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/prompt_caching_5m/test_azure.py + ^^^^^^^^^^^^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +def test_prompt_caching_5m_azure(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert the + upstream usage block surfaces a non-zero cache token count.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + ) + + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens; " + "expected cache_control on the system prompt to produce a non-zero " + "cache_creation_input_tokens or cache_read_input_tokens" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py new file mode 100644 index 00000000000..681a6ecce10 --- /dev/null +++ b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py @@ -0,0 +1,104 @@ +"""prompt_caching_5m x Bedrock (Converse). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the unified `Converse` API path, and +assert that the upstream's usage block reports either +`cache_creation_input_tokens` or `cache_read_input_tokens` > 0. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +def test_prompt_caching_5m_bedrock_converse(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert the + upstream usage block surfaces a non-zero cache token count.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens; " + "expected cache_control on the system prompt to produce a non-zero " + "cache_creation_input_tokens or cache_read_input_tokens" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py new file mode 100644 index 00000000000..f1a3109b3a1 --- /dev/null +++ b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py @@ -0,0 +1,104 @@ +"""prompt_caching_5m x Bedrock (Invoke). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the legacy `InvokeModel` API path, +and assert that the upstream's usage block reports either +`cache_creation_input_tokens` or `cache_read_input_tokens` > 0. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +def test_prompt_caching_5m_bedrock_invoke(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert the + upstream usage block surfaces a non-zero cache token count.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens; " + "expected cache_control on the system prompt to produce a non-zero " + "cache_creation_input_tokens or cache_read_input_tokens" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py b/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py new file mode 100644 index 00000000000..cc5d337dfbe --- /dev/null +++ b/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py @@ -0,0 +1,104 @@ +"""prompt_caching_5m x Vertex AI. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Anthropic's models on Google Cloud Vertex AI, and +assert that the upstream's usage block reports either +`cache_creation_input_tokens` or `cache_read_input_tokens` > 0. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +def test_prompt_caching_5m_vertex_ai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert the + upstream usage block surfaces a non-zero cache token count.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + ) + + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens; " + "expected cache_control on the system prompt to produce a non-zero " + "cache_creation_input_tokens or cache_read_input_tokens" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/rate_limiter.py b/tests/e2e/claude_code/rate_limiter.py new file mode 100644 index 00000000000..06d21b83832 --- /dev/null +++ b/tests/e2e/claude_code/rate_limiter.py @@ -0,0 +1,358 @@ +"""Cross-process token-bucket rate limiter for the Claude Code compat suite. + +The compat matrix runs 75 live `claude` CLI invocations (25 cells × 3 +Claude tiers per cell). When pytest-xdist fans these out across worker +processes, each worker would maintain its own in-memory rate limiter +and the *aggregate* request rate hitting any one upstream provider +would be `workers × per-worker rate` — exactly the situation that +trips Anthropic / Azure / Bedrock / Vertex 429s in the middle of a +matrix run and silently flips green cells red. + +The fix is a **shared, cross-process** token bucket per provider, +backed by a small JSON state file and an OS-level `flock`. Each +`run_claude` invocation acquires one token (sleeping if the bucket is +empty) before launching the CLI; refills happen lazily based on wall +time, so workers can be killed and restarted without losing or +double-spending budget. + +Configuration is driven entirely by environment variables so a +binary-search workflow can shift per-provider rates without code +edits: + + LITELLM_COMPAT_RATE_ANTHROPIC (req/s, default 5.0) + LITELLM_COMPAT_RATE_AZURE (req/s, default 5.0) + LITELLM_COMPAT_RATE_VERTEX_AI (req/s, default 5.0) + LITELLM_COMPAT_RATE_BEDROCK_CONVERSE (req/s, default 5.0) + LITELLM_COMPAT_RATE_BEDROCK_INVOKE (req/s, default 5.0) + LITELLM_COMPAT_RATE_BURST (per-bucket burst override; + default = rate) + LITELLM_COMPAT_RATE_STATE_DIR (state file directory; + default = $TMPDIR/litellm-claude-compat-ratelimit) + +A rate of 0 (or any non-positive value) disables throttling for that +provider — useful when you trust the upstream to handle the burst or +when running unit-test-shaped workloads that never actually hit the +network. + +The provider id is inferred from the model id by `infer_provider`, +mirroring the matrix's column layout (`anthropic`, `azure`, +`vertex_ai`, `bedrock_converse`, `bedrock_invoke`). +""" + +from __future__ import annotations + +import contextlib +import json +import math +import os +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Iterator, Mapping, Optional + +# `fcntl` is POSIX-only; the suite is Linux/macOS only, so we don't +# attempt a Windows fallback. Importing at module load fails fast on +# the (currently non-existent) Windows runner so we don't silently +# degrade to no-locking behavior. +import fcntl + +PROVIDER_ANTHROPIC = "anthropic" +PROVIDER_AZURE = "azure" +PROVIDER_VERTEX_AI = "vertex_ai" +PROVIDER_BEDROCK_CONVERSE = "bedrock_converse" +PROVIDER_BEDROCK_INVOKE = "bedrock_invoke" + +ALL_PROVIDERS = ( + PROVIDER_ANTHROPIC, + PROVIDER_AZURE, + PROVIDER_VERTEX_AI, + PROVIDER_BEDROCK_CONVERSE, + PROVIDER_BEDROCK_INVOKE, +) + +DEFAULT_RATE = 5.0 # req/s per provider, conservative starting point +RATE_ENV_PREFIX = "LITELLM_COMPAT_RATE_" +BURST_ENV = "LITELLM_COMPAT_RATE_BURST" +STATE_DIR_ENV = "LITELLM_COMPAT_RATE_STATE_DIR" +DEFAULT_STATE_DIR_NAME = "litellm-claude-compat-ratelimit" + + +def infer_provider(model: str) -> str: + """Map a model alias to its compat-matrix provider id. + + The matrix column layout is fixed; aliases registered in the proxy + encode the provider via a suffix (`-bedrock-converse`, + `-bedrock-invoke`, `-azure`, `-vertex`) or its absence (Anthropic). + Order matters: the bedrock suffixes both contain `bedrock`, so we + test the more-specific ones first. + """ + if not model: + raise ValueError("model must be a non-empty string") + lower = model.lower() + if lower.endswith("-bedrock-converse"): + return PROVIDER_BEDROCK_CONVERSE + if lower.endswith("-bedrock-invoke"): + return PROVIDER_BEDROCK_INVOKE + if lower.endswith("-azure"): + return PROVIDER_AZURE + if lower.endswith("-vertex"): + return PROVIDER_VERTEX_AI + return PROVIDER_ANTHROPIC + + +@dataclass(frozen=True) +class ProviderConfig: + """Static config snapshot for one provider's bucket. + + Captured up front (rather than re-read per acquire) so the limiter's + behavior in a single process is stable even if env vars are mutated + mid-run. A fresh `RateLimiter` picks up env changes on construction. + """ + + rate_per_sec: float + burst: float + + @property + def enabled(self) -> bool: + return self.rate_per_sec > 0 and self.burst > 0 + + +def load_config( + env: Optional[Mapping[str, str]] = None, +) -> Dict[str, ProviderConfig]: + """Build a {provider: ProviderConfig} from env, applying defaults. + + Parsing failures fall back to the default rate rather than + crashing the test session — a typo in `LITELLM_COMPAT_RATE_AZURE` + should not silently disable throttling, but it also shouldn't + abort 75 live tests with a `ValueError` ten minutes in. + """ + src = env if env is not None else os.environ + + def _as_float(value: Optional[str], default: float) -> float: + if value is None or value == "": + return default + try: + return float(value) + except (TypeError, ValueError): + return default + + burst_override = _as_float(src.get(BURST_ENV), -1.0) + + out: Dict[str, ProviderConfig] = {} + for provider in ALL_PROVIDERS: + env_key = RATE_ENV_PREFIX + provider.upper() + rate = _as_float(src.get(env_key), DEFAULT_RATE) + burst = burst_override if burst_override > 0 else max(rate, 1.0) + out[provider] = ProviderConfig(rate_per_sec=rate, burst=burst) + return out + + +def _state_dir(env: Optional[Mapping[str, str]] = None) -> Path: + """Resolve the directory holding per-provider state files. + + A user-supplied `LITELLM_COMPAT_RATE_STATE_DIR` wins for tests and + container environments where `$TMPDIR` may be ephemeral or shared + in surprising ways. The directory is created lazily with + `parents=True, exist_ok=True` so first-run setup needs no + fixture wiring. + """ + src = env if env is not None else os.environ + explicit = src.get(STATE_DIR_ENV) + if explicit: + return Path(explicit) + return Path(tempfile.gettempdir()) / DEFAULT_STATE_DIR_NAME + + +class RateLimiter: + """Cross-process token bucket per provider. + + Each `acquire(provider)` call: + 1. Opens (creating if needed) `/.json`. + 2. Holds an exclusive `flock` while reading + updating the + {tokens, last_refill} state. + 3. Refills tokens based on `now - last_refill`, capped at burst. + 4. If tokens >= 1, subtracts one and returns immediately. + 5. Otherwise, computes the wall-time delay needed to earn a + single token at the configured rate, releases the lock, and + sleeps. After sleeping it retries — staying under the lock + while sleeping would serialize all workers behind whichever + one held it longest. + + This bounds the *aggregate* req/s seen by the upstream, regardless + of how many xdist workers, threads, or processes are concurrently + running tests against the same provider. + + A `_clock` / `_sleep` injection seam keeps the unit tests fast and + deterministic; production callers should never override either. + """ + + def __init__( + self, + config: Optional[Mapping[str, ProviderConfig]] = None, + state_dir: Optional[Path] = None, + clock: Optional[callable] = None, + sleep: Optional[callable] = None, + ) -> None: + self._config = dict(config) if config is not None else load_config() + self._state_dir = Path(state_dir) if state_dir is not None else _state_dir() + self._clock = clock or time.monotonic + self._sleep = sleep or time.sleep + # `_state_dir.mkdir` once on construction is fine; concurrent + # workers all racing to create the same directory is benign. + self._state_dir.mkdir(parents=True, exist_ok=True) + + def acquire(self, provider: str) -> float: + """Block until one token is available for `provider`. + + Returns the cumulative wall-time spent waiting (0.0 when the + bucket had budget and we returned immediately). Callers can + log this to attribute slow cells to throttling vs. upstream + latency — same role `DriverResult.duration_ms` plays for the + actual CLI invocation. + """ + cfg = self._config.get(provider) + if cfg is None or not cfg.enabled: + return 0.0 + + path = self._state_path(provider) + total_waited = 0.0 + while True: + now = self._clock() + sleep_for = self._try_consume(path, cfg, now) + if sleep_for <= 0: + return total_waited + self._sleep(sleep_for) + total_waited += sleep_for + + def _state_path(self, provider: str) -> Path: + return self._state_dir / f"{provider}.json" + + def _try_consume(self, path: Path, cfg: ProviderConfig, now: float) -> float: + """Atomically refill and try to take one token. + + Returns 0.0 if a token was consumed, or a positive sleep + duration (seconds) if the caller must wait before retrying. + + We hold an exclusive `flock` only across the read-modify-write + of the JSON state — never across a `sleep` — so workers don't + serialize while one of them is parked. + """ + # `os.open` + `os.O_CREAT | os.O_RDWR` gives us a fd we can + # both lock and read/write through. Opening with `"a+"` then + # seeking is equivalent but uglier; this version is closer to + # the canonical flock recipe. + fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o600) + try: + fcntl.flock(fd, fcntl.LOCK_EX) + try: + tokens, last_refill = self._read_state(fd, cfg, now) + tokens, last_refill = self._refill(tokens, last_refill, now, cfg) + if tokens >= 1.0: + tokens -= 1.0 + self._write_state(fd, tokens, last_refill) + return 0.0 + # Not enough budget. Persist the refilled state so a + # subsequent caller doesn't have to redo the math, then + # release the lock and tell the caller how long to + # sleep before retrying. + self._write_state(fd, tokens, last_refill) + deficit = 1.0 - tokens + # `deficit / rate` seconds will earn exactly enough + # for one token. Add a tiny safety margin so we don't + # wake up nanoseconds early and spin. + return (deficit / cfg.rate_per_sec) + 1e-3 + finally: + fcntl.flock(fd, fcntl.LOCK_UN) + finally: + os.close(fd) + + @staticmethod + def _read_state(fd: int, cfg: ProviderConfig, now: float) -> tuple: + """Read {tokens, last_refill} from `fd`, defaulting to a full + bucket on a missing/empty/corrupt file. + + New files start full so the very first request never waits; + corrupt files are treated like new files because the + alternative — refusing to run — is worse than briefly + over-spending one bucket's worth of budget. + """ + os.lseek(fd, 0, os.SEEK_SET) + raw = os.read(fd, 4096).decode("utf-8") + if not raw.strip(): + return cfg.burst, now + try: + obj = json.loads(raw) + tokens = float(obj.get("tokens", cfg.burst)) + last_refill = float(obj.get("last_refill", now)) + return tokens, last_refill + except (ValueError, TypeError): + return cfg.burst, now + + @staticmethod + def _refill( + tokens: float, last_refill: float, now: float, cfg: ProviderConfig + ) -> tuple: + """Apply elapsed time to the bucket, capped at burst. + + Negative elapsed (clock went backward, e.g. across host + sleep/resume or a manually-tweaked monotonic mock) is clamped + to zero so we never *remove* tokens. + """ + elapsed = max(0.0, now - last_refill) + tokens = min(cfg.burst, tokens + elapsed * cfg.rate_per_sec) + return tokens, now + + @staticmethod + def _write_state(fd: int, tokens: float, last_refill: float) -> None: + payload = json.dumps({"tokens": tokens, "last_refill": last_refill}).encode( + "utf-8" + ) + os.lseek(fd, 0, os.SEEK_SET) + os.ftruncate(fd, 0) + os.write(fd, payload) + + +# A single process-wide instance is all we need: provider-keyed state +# is stored in files, so multiple `RateLimiter` instances would just +# duplicate the in-process bookkeeping. We expose a getter rather than +# the instance directly so unit tests can install a custom limiter +# scoped to a tmp directory without monkeypatching globals. +_default: Optional[RateLimiter] = None + + +def get_default_limiter() -> RateLimiter: + global _default + if _default is None: + _default = RateLimiter() + return _default + + +def reset_default_limiter() -> None: + """Drop the cached default limiter; the next `get_default_limiter` + call rebuilds it from the current environment. + + Useful between unit tests that patch env vars: without this they'd + keep reading the stale config snapshot from the first call. + """ + global _default + _default = None + + +@contextlib.contextmanager +def use_limiter(limiter: RateLimiter) -> Iterator[RateLimiter]: + """Temporarily install `limiter` as the process default. + + The driver's `run_claude` calls `get_default_limiter()`; tests that + want a controlled tmp-dir-backed limiter use this contextmanager + to swap one in without touching env vars or the on-disk default + state. + """ + global _default + previous = _default + _default = limiter + try: + yield limiter + finally: + _default = previous diff --git a/tests/e2e/claude_code/run_compat.sh b/tests/e2e/claude_code/run_compat.sh new file mode 100755 index 00000000000..4d8d0b6d7b2 --- /dev/null +++ b/tests/e2e/claude_code/run_compat.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Run the Claude Code compat matrix end-to-end against a live LiteLLM +# proxy, with per-provider rate limits applied via the cross-process +# token bucket in `tests/e2e/claude_code/rate_limiter.py`. +# +# Designed for binary-searching the ideal X / Y / Z req/s per provider: +# 1. Pick an initial rate (e.g. 5/s for everyone). +# 2. Run this script. +# 3. Read `compat-rate-limit-summary.json` to see whether any provider +# hit a 429-shaped error during the run. +# 4. If a provider has `rate_limited > 0`, halve its rate; else, double it. +# 5. Repeat until the highest no-429 rate is found. +# +# Required env (proxy connection): +# LITELLM_PROXY_BASE_URL e.g. http://localhost:4000 +# LITELLM_PROXY_API_KEY e.g. sk-1234 +# +# Optional env (rate limits, all default to 5 req/s; 0 disables a column): +# LITELLM_COMPAT_RATE_ANTHROPIC +# LITELLM_COMPAT_RATE_AZURE +# LITELLM_COMPAT_RATE_VERTEX_AI +# LITELLM_COMPAT_RATE_BEDROCK_CONVERSE +# LITELLM_COMPAT_RATE_BEDROCK_INVOKE +# LITELLM_COMPAT_RATE_BURST override per-bucket burst +# +# Optional env (parallelism): +# COMPAT_XDIST_WORKERS passed to `pytest -n` (default: auto) +# +# Optional env (artifacts): +# COMPAT_RESULTS_PATH default: compat-results.json +# COMPAT_RATE_LIMIT_SUMMARY_PATH default: compat-rate-limit-summary.json + +set -euo pipefail + +if [[ -z "${LITELLM_PROXY_BASE_URL:-}" || -z "${LITELLM_PROXY_API_KEY:-}" ]]; then + echo "error: LITELLM_PROXY_BASE_URL and LITELLM_PROXY_API_KEY must be set" >&2 + exit 64 +fi + +# Reset the cross-process rate-limiter state from any prior run. Stale +# token-bucket files would let a previous run's accumulated budget bleed +# into the new one, which subtly biases the binary search. +state_dir="${LITELLM_COMPAT_RATE_STATE_DIR:-${TMPDIR:-/tmp}/litellm-claude-compat-ratelimit}" +if [[ -d "$state_dir" ]]; then + rm -rf "$state_dir" +fi + +# Worker count. `auto` picks one worker per CPU; the rate limiter +# enforces aggregate provider rates regardless of worker count, so +# this is a "go as fast as the limiter allows" knob, not a tuning knob. +workers="${COMPAT_XDIST_WORKERS:-auto}" + +# Where the artifacts land. We resolve them now so the summary file is +# always at a known path the caller can grep, even if they didn't set +# the env explicitly. +results_path="${COMPAT_RESULTS_PATH:-compat-results.json}" +summary_path="${COMPAT_RATE_LIMIT_SUMMARY_PATH:-compat-rate-limit-summary.json}" + +echo "[run_compat] rates:" +for provider in ANTHROPIC AZURE VERTEX_AI BEDROCK_CONVERSE BEDROCK_INVOKE; do + var="LITELLM_COMPAT_RATE_${provider}" + echo " ${provider}=${!var:-default(5/s)}" +done +echo " BURST=${LITELLM_COMPAT_RATE_BURST:-default(=rate)}" +echo "[run_compat] xdist workers: ${workers}" +echo "[run_compat] results: ${results_path}" +echo "[run_compat] summary: ${summary_path}" + +# Run only the per-feature live tests; skip the unit-test directories +# (they're under directories starting with `_`). The dist=loadfile +# scheduler keeps each test file pinned to a single worker, which is +# what we want — every test in a file shares a single ThreadPoolExecutor +# fanout, and we don't gain anything by splitting it across workers. +start=$(date +%s) +set +e +COMPAT_RESULTS_PATH="${results_path}" \ +COMPAT_RATE_LIMIT_SUMMARY_PATH="${summary_path}" \ +PATH="$HOME/.local/bin:$PATH" \ +uv run pytest \ + tests/e2e/claude_code/basic_messaging_non_streaming \ + tests/e2e/claude_code/basic_messaging_streaming \ + tests/e2e/claude_code/thinking \ + tests/e2e/claude_code/tool_use \ + tests/e2e/claude_code/vision \ + tests/e2e/claude_code/prompt_caching_5m \ + -n "${workers}" \ + --dist=loadfile \ + -q \ + "$@" + +exit_code=$? +set -e +end=$(date +%s) +echo "[run_compat] wall time: $((end - start))s" + +# Surface the rate-limit summary inline so a human reader doesn't have +# to `cat` the JSON file. The full file is still on disk for the binary +# search loop. +if [[ -f "${summary_path}" ]]; then + echo "[run_compat] summary: ${summary_path}" + if command -v jq >/dev/null 2>&1; then + jq '.totals, .per_provider' "${summary_path}" + else + cat "${summary_path}" + fi +fi + +exit "${exit_code}" diff --git a/tests/e2e/claude_code/sample_compatibility-matrix.json b/tests/e2e/claude_code/sample_compatibility-matrix.json new file mode 100644 index 00000000000..cfc7f3885d3 --- /dev/null +++ b/tests/e2e/claude_code/sample_compatibility-matrix.json @@ -0,0 +1,141 @@ +{ + "schema_version": "1", + "generated_at": "2026-04-25T00:00:00Z", + "litellm_version": "v1.83.0-stable", + "claude_code_version": "2.1.120", + "providers": [ + "anthropic", + "bedrock_invoke", + "bedrock_converse", + "vertex_ai", + "azure" + ], + "features": [ + { + "id": "basic_messaging_non_streaming", + "name": "Basic messaging (non-streaming)", + "providers": { + "anthropic": { + "status": "pass" + }, + "bedrock_invoke": { + "status": "pass" + }, + "bedrock_converse": { + "status": "pass" + }, + "vertex_ai": { + "status": "pass" + }, + "azure": { + "status": "pass" + } + } + }, + { + "id": "basic_messaging_streaming", + "name": "Basic messaging (streaming)", + "providers": { + "anthropic": { + "status": "pass" + }, + "bedrock_invoke": { + "status": "pass" + }, + "bedrock_converse": { + "status": "pass" + }, + "vertex_ai": { + "status": "pass" + }, + "azure": { + "status": "pass" + } + } + }, + { + "id": "tool_use", + "name": "Tool use", + "providers": { + "anthropic": { + "status": "pass" + }, + "bedrock_invoke": { + "status": "pass" + }, + "bedrock_converse": { + "status": "pass" + }, + "vertex_ai": { + "status": "pass" + }, + "azure": { + "status": "pass" + } + } + }, + { + "id": "prompt_caching_5m", + "name": "Prompt caching (5m TTL)", + "providers": { + "anthropic": { + "status": "pass" + }, + "bedrock_invoke": { + "status": "pass" + }, + "bedrock_converse": { + "status": "pass" + }, + "vertex_ai": { + "status": "pass" + }, + "azure": { + "status": "pass" + } + } + }, + { + "id": "vision", + "name": "Vision", + "providers": { + "anthropic": { + "status": "pass" + }, + "bedrock_invoke": { + "status": "pass" + }, + "bedrock_converse": { + "status": "pass" + }, + "vertex_ai": { + "status": "pass" + }, + "azure": { + "status": "pass" + } + } + }, + { + "id": "thinking", + "name": "Thinking", + "providers": { + "anthropic": { + "status": "pass" + }, + "bedrock_invoke": { + "status": "pass" + }, + "bedrock_converse": { + "status": "pass" + }, + "vertex_ai": { + "status": "pass" + }, + "azure": { + "status": "pass" + } + } + } + ] +} diff --git a/tests/e2e/claude_code/structured_outputs/__init__.py b/tests/e2e/claude_code/structured_outputs/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/structured_outputs/test_anthropic.py b/tests/e2e/claude_code/structured_outputs/test_anthropic.py new file mode 100644 index 00000000000..610d8433b72 --- /dev/null +++ b/tests/e2e/claude_code/structured_outputs/test_anthropic.py @@ -0,0 +1,220 @@ +"""structured_outputs x Anthropic. + +Drive the real `claude` CLI in headless mode with the `--json-schema` +flag, route through a LiteLLM proxy aimed at Anthropic, and assert that +the final stream-json `result` event surfaces a `structured_output` +object whose shape matches the schema. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/structured_outputs/test_anthropic.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider + +What this row actually exercises (and what it does not): + +`--json-schema` is implemented client-side by Claude Code: the CLI +synthesizes a synthetic `StructuredOutput` tool whose `input_schema` +equals the user-supplied JSON Schema, forces the model toward it, and +finally extracts the tool_use input on the trailing `result` event as +`structured_output: {...}`. The proxy never sees `output_config.schema` +in this flow -- it sees a normal `tools` array with one synthetic +tool. + +This makes the row a tool-use feature test in disguise. It's still a +distinct row from `tool_use` because: + + - The synthetic tool is generated per request from a user schema, not + a developer-declared one. Provider-side bugs that special-case + `Claude Code`-generated tool names (e.g. case-folding `tool_use` + blocks back to lowercase, or stripping the StructuredOutput-only + `additionalProperties: false`) only surface here. + - The success signal lives on the *final* `result` event, not the + intermediate `assistant` events the `tool_use` row checks. A proxy + that drops trailing events (seen in early Bedrock Converse SSE + plumbing) breaks this cell while leaving `tool_use` green. + +It is NOT a test of Anthropic's server-side `output_config.schema` +parameter -- that's a different feature used internally by Claude Code +for session-title generation and is not reachable from any CLI flag. +LiteLLM's `output_config`-stripping fixes (2.1.122, 2.1.81) surface in +the `count_tokens` and other HTTP-probe rows, not here. + +Three Claude tiers run in parallel; one `compat_result.add(...)` per +tier so the matrix's "all three must pass" rule applies. +""" + +from __future__ import annotations + +import json +import os +import re +from typing import Any, Mapping, Optional, Sequence, Tuple + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +# Minimal schema with one required integer field. Kept intentionally +# small -- the matrix tests the *plumbing*, not the model's ability to +# satisfy a complex schema. A trivial arithmetic prompt + a one-field +# integer schema gives every tier (including Haiku) enough headroom +# that schema satisfaction is essentially deterministic, isolating +# failures to the proxy / transport. +SCHEMA = { + "type": "object", + "properties": {"answer": {"type": "integer"}}, + "required": ["answer"], + "additionalProperties": False, +} +SCHEMA_JSON = json.dumps(SCHEMA, separators=(",", ":")) + +# A prompt the model has no reason to misanswer; we don't check the +# value, but a wrong answer would suggest the structured-output +# pathway is silently degrading reasoning, which is itself worth +# noticing. +PROMPT = "What is 2 + 2? Reply only via the structured output." + + +def _extract_structured_output( + events: Sequence[Mapping[str, Any]], +) -> Optional[Mapping[str, Any]]: + """Return the `structured_output` payload from the last `result` event. + + Claude Code emits its terminal stream-json line as + `{"type":"result","structured_output":{...},...}` when a request + used `--json-schema` and the model actually produced a valid tool + call. If the model bailed or the proxy ate the trailing events, + `structured_output` is missing -- which is exactly the failure + mode we want this row to surface, so the caller treats `None` as + "feature did not work end-to-end". + """ + for event in reversed(list(events)): + if event.get("type") != "result": + continue + so = event.get("structured_output") + if isinstance(so, Mapping): + return so + return None + + +def _validate_against_schema( + payload: Mapping[str, Any], schema: Mapping[str, Any] +) -> Optional[str]: + """Tiny shape validator covering the subset we actually need. + + We deliberately do not pull in `jsonschema` as a test dep: the + matrix's success signal is "does the proxy let the synthetic + StructuredOutput tool round-trip end-to-end", and that's + answerable with a presence + type check over `required` keys. + Any malformed schema beyond that would be a Claude Code bug, + not a LiteLLM-proxy bug, so a deeper check would only add false + failures on the wrong axis. + """ + type_map = { + "integer": int, + "number": (int, float), + "string": str, + "boolean": bool, + "array": list, + "object": Mapping, + } + required = schema.get("required") or [] + properties = schema.get("properties") or {} + for key in required: + if key not in payload: + return f"missing required key {key!r}" + expected = (properties.get(key) or {}).get("type") + if expected and expected in type_map: + if not isinstance(payload[key], type_map[expected]): + return ( + f"key {key!r} has wrong type: " + f"expected {expected}, got {type(payload[key]).__name__}" + ) + # bool is a subclass of int in Python; reject `True`/`False` + # when the schema asked for an integer/number. + if expected in ("integer", "number") and isinstance(payload[key], bool): + return f"key {key!r} is a bool but schema asked for {expected}" + return None + + +def test_structured_outputs_anthropic(compat_result): + """Drive `claude --json-schema ...` against the LiteLLM proxy and + assert the trailing `result` event contains a schema-conforming + `structured_output`.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt=PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=["--json-schema", SCHEMA_JSON], + ) + + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + payload = _extract_structured_output(outcome.events) + if payload is None: + error = ( + f"[{model}] no `structured_output` in trailing result event; " + "Claude Code's StructuredOutput tool round-trip did not " + "complete end-to-end through the proxy" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + shape_error = _validate_against_schema(payload, SCHEMA) + if shape_error is not None: + error = f"[{model}] structured_output shape error: {shape_error}; payload={payload}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/structured_outputs/test_azure.py b/tests/e2e/claude_code/structured_outputs/test_azure.py new file mode 100644 index 00000000000..290f9156910 --- /dev/null +++ b/tests/e2e/claude_code/structured_outputs/test_azure.py @@ -0,0 +1,220 @@ +"""structured_outputs x Azure (Microsoft Foundry). + +Drive the real `claude` CLI in headless mode with the `--json-schema` +flag, route through a LiteLLM proxy aimed at Azure (Microsoft Foundry), and assert that +the final stream-json `result` event surfaces a `structured_output` +object whose shape matches the schema. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/structured_outputs/test_azure.py + ^^^^^^^^^^^^^^^^^^ ^^^^^ + feature_id provider + +What this row actually exercises (and what it does not): + +`--json-schema` is implemented client-side by Claude Code: the CLI +synthesizes a synthetic `StructuredOutput` tool whose `input_schema` +equals the user-supplied JSON Schema, forces the model toward it, and +finally extracts the tool_use input on the trailing `result` event as +`structured_output: {...}`. The proxy never sees `output_config.schema` +in this flow -- it sees a normal `tools` array with one synthetic +tool. + +This makes the row a tool-use feature test in disguise. It's still a +distinct row from `tool_use` because: + + - The synthetic tool is generated per request from a user schema, not + a developer-declared one. Provider-side bugs that special-case + `Claude Code`-generated tool names (e.g. case-folding `tool_use` + blocks back to lowercase, or stripping the StructuredOutput-only + `additionalProperties: false`) only surface here. + - The success signal lives on the *final* `result` event, not the + intermediate `assistant` events the `tool_use` row checks. A proxy + that drops trailing events (seen in early Bedrock Converse SSE + plumbing) breaks this cell while leaving `tool_use` green. + +It is NOT a test of Anthropic's server-side `output_config.schema` +parameter -- that's a different feature used internally by Claude Code +for session-title generation and is not reachable from any CLI flag. +LiteLLM's `output_config`-stripping fixes (2.1.122, 2.1.81) surface in +the `count_tokens` and other HTTP-probe rows, not here. + +Three Claude tiers run in parallel; one `compat_result.add(...)` per +tier so the matrix's "all three must pass" rule applies. +""" + +from __future__ import annotations + +import json +import os +import re +from typing import Any, Mapping, Optional, Sequence, Tuple + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + +# Minimal schema with one required integer field. Kept intentionally +# small -- the matrix tests the *plumbing*, not the model's ability to +# satisfy a complex schema. A trivial arithmetic prompt + a one-field +# integer schema gives every tier (including Haiku) enough headroom +# that schema satisfaction is essentially deterministic, isolating +# failures to the proxy / transport. +SCHEMA = { + "type": "object", + "properties": {"answer": {"type": "integer"}}, + "required": ["answer"], + "additionalProperties": False, +} +SCHEMA_JSON = json.dumps(SCHEMA, separators=(",", ":")) + +# A prompt the model has no reason to misanswer; we don't check the +# value, but a wrong answer would suggest the structured-output +# pathway is silently degrading reasoning, which is itself worth +# noticing. +PROMPT = "What is 2 + 2? Reply only via the structured output." + + +def _extract_structured_output( + events: Sequence[Mapping[str, Any]], +) -> Optional[Mapping[str, Any]]: + """Return the `structured_output` payload from the last `result` event. + + Claude Code emits its terminal stream-json line as + `{"type":"result","structured_output":{...},...}` when a request + used `--json-schema` and the model actually produced a valid tool + call. If the model bailed or the proxy ate the trailing events, + `structured_output` is missing -- which is exactly the failure + mode we want this row to surface, so the caller treats `None` as + "feature did not work end-to-end". + """ + for event in reversed(list(events)): + if event.get("type") != "result": + continue + so = event.get("structured_output") + if isinstance(so, Mapping): + return so + return None + + +def _validate_against_schema( + payload: Mapping[str, Any], schema: Mapping[str, Any] +) -> Optional[str]: + """Tiny shape validator covering the subset we actually need. + + We deliberately do not pull in `jsonschema` as a test dep: the + matrix's success signal is "does the proxy let the synthetic + StructuredOutput tool round-trip end-to-end", and that's + answerable with a presence + type check over `required` keys. + Any malformed schema beyond that would be a Claude Code bug, + not a LiteLLM-proxy bug, so a deeper check would only add false + failures on the wrong axis. + """ + type_map = { + "integer": int, + "number": (int, float), + "string": str, + "boolean": bool, + "array": list, + "object": Mapping, + } + required = schema.get("required") or [] + properties = schema.get("properties") or {} + for key in required: + if key not in payload: + return f"missing required key {key!r}" + expected = (properties.get(key) or {}).get("type") + if expected and expected in type_map: + if not isinstance(payload[key], type_map[expected]): + return ( + f"key {key!r} has wrong type: " + f"expected {expected}, got {type(payload[key]).__name__}" + ) + # bool is a subclass of int in Python; reject `True`/`False` + # when the schema asked for an integer/number. + if expected in ("integer", "number") and isinstance(payload[key], bool): + return f"key {key!r} is a bool but schema asked for {expected}" + return None + + +def test_structured_outputs_azure(compat_result): + """Drive `claude --json-schema ...` against the LiteLLM proxy and + assert the trailing `result` event contains a schema-conforming + `structured_output`.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt=PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=["--json-schema", SCHEMA_JSON], + ) + + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + payload = _extract_structured_output(outcome.events) + if payload is None: + error = ( + f"[{model}] no `structured_output` in trailing result event; " + "Claude Code's StructuredOutput tool round-trip did not " + "complete end-to-end through the proxy" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + shape_error = _validate_against_schema(payload, SCHEMA) + if shape_error is not None: + error = f"[{model}] structured_output shape error: {shape_error}; payload={payload}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py b/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py new file mode 100644 index 00000000000..5179014773c --- /dev/null +++ b/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py @@ -0,0 +1,220 @@ +"""structured_outputs x Bedrock (Converse). + +Drive the real `claude` CLI in headless mode with the `--json-schema` +flag, route through a LiteLLM proxy aimed at Bedrock (Converse), and assert that +the final stream-json `result` event surfaces a `structured_output` +object whose shape matches the schema. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider + +What this row actually exercises (and what it does not): + +`--json-schema` is implemented client-side by Claude Code: the CLI +synthesizes a synthetic `StructuredOutput` tool whose `input_schema` +equals the user-supplied JSON Schema, forces the model toward it, and +finally extracts the tool_use input on the trailing `result` event as +`structured_output: {...}`. The proxy never sees `output_config.schema` +in this flow -- it sees a normal `tools` array with one synthetic +tool. + +This makes the row a tool-use feature test in disguise. It's still a +distinct row from `tool_use` because: + + - The synthetic tool is generated per request from a user schema, not + a developer-declared one. Provider-side bugs that special-case + `Claude Code`-generated tool names (e.g. case-folding `tool_use` + blocks back to lowercase, or stripping the StructuredOutput-only + `additionalProperties: false`) only surface here. + - The success signal lives on the *final* `result` event, not the + intermediate `assistant` events the `tool_use` row checks. A proxy + that drops trailing events (seen in early Bedrock Converse SSE + plumbing) breaks this cell while leaving `tool_use` green. + +It is NOT a test of Anthropic's server-side `output_config.schema` +parameter -- that's a different feature used internally by Claude Code +for session-title generation and is not reachable from any CLI flag. +LiteLLM's `output_config`-stripping fixes (2.1.122, 2.1.81) surface in +the `count_tokens` and other HTTP-probe rows, not here. + +Three Claude tiers run in parallel; one `compat_result.add(...)` per +tier so the matrix's "all three must pass" rule applies. +""" + +from __future__ import annotations + +import json +import os +import re +from typing import Any, Mapping, Optional, Sequence, Tuple + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + +# Minimal schema with one required integer field. Kept intentionally +# small -- the matrix tests the *plumbing*, not the model's ability to +# satisfy a complex schema. A trivial arithmetic prompt + a one-field +# integer schema gives every tier (including Haiku) enough headroom +# that schema satisfaction is essentially deterministic, isolating +# failures to the proxy / transport. +SCHEMA = { + "type": "object", + "properties": {"answer": {"type": "integer"}}, + "required": ["answer"], + "additionalProperties": False, +} +SCHEMA_JSON = json.dumps(SCHEMA, separators=(",", ":")) + +# A prompt the model has no reason to misanswer; we don't check the +# value, but a wrong answer would suggest the structured-output +# pathway is silently degrading reasoning, which is itself worth +# noticing. +PROMPT = "What is 2 + 2? Reply only via the structured output." + + +def _extract_structured_output( + events: Sequence[Mapping[str, Any]], +) -> Optional[Mapping[str, Any]]: + """Return the `structured_output` payload from the last `result` event. + + Claude Code emits its terminal stream-json line as + `{"type":"result","structured_output":{...},...}` when a request + used `--json-schema` and the model actually produced a valid tool + call. If the model bailed or the proxy ate the trailing events, + `structured_output` is missing -- which is exactly the failure + mode we want this row to surface, so the caller treats `None` as + "feature did not work end-to-end". + """ + for event in reversed(list(events)): + if event.get("type") != "result": + continue + so = event.get("structured_output") + if isinstance(so, Mapping): + return so + return None + + +def _validate_against_schema( + payload: Mapping[str, Any], schema: Mapping[str, Any] +) -> Optional[str]: + """Tiny shape validator covering the subset we actually need. + + We deliberately do not pull in `jsonschema` as a test dep: the + matrix's success signal is "does the proxy let the synthetic + StructuredOutput tool round-trip end-to-end", and that's + answerable with a presence + type check over `required` keys. + Any malformed schema beyond that would be a Claude Code bug, + not a LiteLLM-proxy bug, so a deeper check would only add false + failures on the wrong axis. + """ + type_map = { + "integer": int, + "number": (int, float), + "string": str, + "boolean": bool, + "array": list, + "object": Mapping, + } + required = schema.get("required") or [] + properties = schema.get("properties") or {} + for key in required: + if key not in payload: + return f"missing required key {key!r}" + expected = (properties.get(key) or {}).get("type") + if expected and expected in type_map: + if not isinstance(payload[key], type_map[expected]): + return ( + f"key {key!r} has wrong type: " + f"expected {expected}, got {type(payload[key]).__name__}" + ) + # bool is a subclass of int in Python; reject `True`/`False` + # when the schema asked for an integer/number. + if expected in ("integer", "number") and isinstance(payload[key], bool): + return f"key {key!r} is a bool but schema asked for {expected}" + return None + + +def test_structured_outputs_bedrock_converse(compat_result): + """Drive `claude --json-schema ...` against the LiteLLM proxy and + assert the trailing `result` event contains a schema-conforming + `structured_output`.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt=PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=["--json-schema", SCHEMA_JSON], + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + payload = _extract_structured_output(outcome.events) + if payload is None: + error = ( + f"[{model}] no `structured_output` in trailing result event; " + "Claude Code's StructuredOutput tool round-trip did not " + "complete end-to-end through the proxy" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + shape_error = _validate_against_schema(payload, SCHEMA) + if shape_error is not None: + error = f"[{model}] structured_output shape error: {shape_error}; payload={payload}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py b/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py new file mode 100644 index 00000000000..313a714be34 --- /dev/null +++ b/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py @@ -0,0 +1,220 @@ +"""structured_outputs x Bedrock (Invoke). + +Drive the real `claude` CLI in headless mode with the `--json-schema` +flag, route through a LiteLLM proxy aimed at Bedrock (Invoke), and assert that +the final stream-json `result` event surfaces a `structured_output` +object whose shape matches the schema. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider + +What this row actually exercises (and what it does not): + +`--json-schema` is implemented client-side by Claude Code: the CLI +synthesizes a synthetic `StructuredOutput` tool whose `input_schema` +equals the user-supplied JSON Schema, forces the model toward it, and +finally extracts the tool_use input on the trailing `result` event as +`structured_output: {...}`. The proxy never sees `output_config.schema` +in this flow -- it sees a normal `tools` array with one synthetic +tool. + +This makes the row a tool-use feature test in disguise. It's still a +distinct row from `tool_use` because: + + - The synthetic tool is generated per request from a user schema, not + a developer-declared one. Provider-side bugs that special-case + `Claude Code`-generated tool names (e.g. case-folding `tool_use` + blocks back to lowercase, or stripping the StructuredOutput-only + `additionalProperties: false`) only surface here. + - The success signal lives on the *final* `result` event, not the + intermediate `assistant` events the `tool_use` row checks. A proxy + that drops trailing events (seen in early Bedrock Converse SSE + plumbing) breaks this cell while leaving `tool_use` green. + +It is NOT a test of Anthropic's server-side `output_config.schema` +parameter -- that's a different feature used internally by Claude Code +for session-title generation and is not reachable from any CLI flag. +LiteLLM's `output_config`-stripping fixes (2.1.122, 2.1.81) surface in +the `count_tokens` and other HTTP-probe rows, not here. + +Three Claude tiers run in parallel; one `compat_result.add(...)` per +tier so the matrix's "all three must pass" rule applies. +""" + +from __future__ import annotations + +import json +import os +import re +from typing import Any, Mapping, Optional, Sequence, Tuple + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + +# Minimal schema with one required integer field. Kept intentionally +# small -- the matrix tests the *plumbing*, not the model's ability to +# satisfy a complex schema. A trivial arithmetic prompt + a one-field +# integer schema gives every tier (including Haiku) enough headroom +# that schema satisfaction is essentially deterministic, isolating +# failures to the proxy / transport. +SCHEMA = { + "type": "object", + "properties": {"answer": {"type": "integer"}}, + "required": ["answer"], + "additionalProperties": False, +} +SCHEMA_JSON = json.dumps(SCHEMA, separators=(",", ":")) + +# A prompt the model has no reason to misanswer; we don't check the +# value, but a wrong answer would suggest the structured-output +# pathway is silently degrading reasoning, which is itself worth +# noticing. +PROMPT = "What is 2 + 2? Reply only via the structured output." + + +def _extract_structured_output( + events: Sequence[Mapping[str, Any]], +) -> Optional[Mapping[str, Any]]: + """Return the `structured_output` payload from the last `result` event. + + Claude Code emits its terminal stream-json line as + `{"type":"result","structured_output":{...},...}` when a request + used `--json-schema` and the model actually produced a valid tool + call. If the model bailed or the proxy ate the trailing events, + `structured_output` is missing -- which is exactly the failure + mode we want this row to surface, so the caller treats `None` as + "feature did not work end-to-end". + """ + for event in reversed(list(events)): + if event.get("type") != "result": + continue + so = event.get("structured_output") + if isinstance(so, Mapping): + return so + return None + + +def _validate_against_schema( + payload: Mapping[str, Any], schema: Mapping[str, Any] +) -> Optional[str]: + """Tiny shape validator covering the subset we actually need. + + We deliberately do not pull in `jsonschema` as a test dep: the + matrix's success signal is "does the proxy let the synthetic + StructuredOutput tool round-trip end-to-end", and that's + answerable with a presence + type check over `required` keys. + Any malformed schema beyond that would be a Claude Code bug, + not a LiteLLM-proxy bug, so a deeper check would only add false + failures on the wrong axis. + """ + type_map = { + "integer": int, + "number": (int, float), + "string": str, + "boolean": bool, + "array": list, + "object": Mapping, + } + required = schema.get("required") or [] + properties = schema.get("properties") or {} + for key in required: + if key not in payload: + return f"missing required key {key!r}" + expected = (properties.get(key) or {}).get("type") + if expected and expected in type_map: + if not isinstance(payload[key], type_map[expected]): + return ( + f"key {key!r} has wrong type: " + f"expected {expected}, got {type(payload[key]).__name__}" + ) + # bool is a subclass of int in Python; reject `True`/`False` + # when the schema asked for an integer/number. + if expected in ("integer", "number") and isinstance(payload[key], bool): + return f"key {key!r} is a bool but schema asked for {expected}" + return None + + +def test_structured_outputs_bedrock_invoke(compat_result): + """Drive `claude --json-schema ...` against the LiteLLM proxy and + assert the trailing `result` event contains a schema-conforming + `structured_output`.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt=PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=["--json-schema", SCHEMA_JSON], + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + payload = _extract_structured_output(outcome.events) + if payload is None: + error = ( + f"[{model}] no `structured_output` in trailing result event; " + "Claude Code's StructuredOutput tool round-trip did not " + "complete end-to-end through the proxy" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + shape_error = _validate_against_schema(payload, SCHEMA) + if shape_error is not None: + error = f"[{model}] structured_output shape error: {shape_error}; payload={payload}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py b/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py new file mode 100644 index 00000000000..ec04c724193 --- /dev/null +++ b/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py @@ -0,0 +1,220 @@ +"""structured_outputs x Vertex AI. + +Drive the real `claude` CLI in headless mode with the `--json-schema` +flag, route through a LiteLLM proxy aimed at Vertex AI, and assert that +the final stream-json `result` event surfaces a `structured_output` +object whose shape matches the schema. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/structured_outputs/test_vertex_ai.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider + +What this row actually exercises (and what it does not): + +`--json-schema` is implemented client-side by Claude Code: the CLI +synthesizes a synthetic `StructuredOutput` tool whose `input_schema` +equals the user-supplied JSON Schema, forces the model toward it, and +finally extracts the tool_use input on the trailing `result` event as +`structured_output: {...}`. The proxy never sees `output_config.schema` +in this flow -- it sees a normal `tools` array with one synthetic +tool. + +This makes the row a tool-use feature test in disguise. It's still a +distinct row from `tool_use` because: + + - The synthetic tool is generated per request from a user schema, not + a developer-declared one. Provider-side bugs that special-case + `Claude Code`-generated tool names (e.g. case-folding `tool_use` + blocks back to lowercase, or stripping the StructuredOutput-only + `additionalProperties: false`) only surface here. + - The success signal lives on the *final* `result` event, not the + intermediate `assistant` events the `tool_use` row checks. A proxy + that drops trailing events (seen in early Bedrock Converse SSE + plumbing) breaks this cell while leaving `tool_use` green. + +It is NOT a test of Anthropic's server-side `output_config.schema` +parameter -- that's a different feature used internally by Claude Code +for session-title generation and is not reachable from any CLI flag. +LiteLLM's `output_config`-stripping fixes (2.1.122, 2.1.81) surface in +the `count_tokens` and other HTTP-probe rows, not here. + +Three Claude tiers run in parallel; one `compat_result.add(...)` per +tier so the matrix's "all three must pass" rule applies. +""" + +from __future__ import annotations + +import json +import os +import re +from typing import Any, Mapping, Optional, Sequence, Tuple + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + +# Minimal schema with one required integer field. Kept intentionally +# small -- the matrix tests the *plumbing*, not the model's ability to +# satisfy a complex schema. A trivial arithmetic prompt + a one-field +# integer schema gives every tier (including Haiku) enough headroom +# that schema satisfaction is essentially deterministic, isolating +# failures to the proxy / transport. +SCHEMA = { + "type": "object", + "properties": {"answer": {"type": "integer"}}, + "required": ["answer"], + "additionalProperties": False, +} +SCHEMA_JSON = json.dumps(SCHEMA, separators=(",", ":")) + +# A prompt the model has no reason to misanswer; we don't check the +# value, but a wrong answer would suggest the structured-output +# pathway is silently degrading reasoning, which is itself worth +# noticing. +PROMPT = "What is 2 + 2? Reply only via the structured output." + + +def _extract_structured_output( + events: Sequence[Mapping[str, Any]], +) -> Optional[Mapping[str, Any]]: + """Return the `structured_output` payload from the last `result` event. + + Claude Code emits its terminal stream-json line as + `{"type":"result","structured_output":{...},...}` when a request + used `--json-schema` and the model actually produced a valid tool + call. If the model bailed or the proxy ate the trailing events, + `structured_output` is missing -- which is exactly the failure + mode we want this row to surface, so the caller treats `None` as + "feature did not work end-to-end". + """ + for event in reversed(list(events)): + if event.get("type") != "result": + continue + so = event.get("structured_output") + if isinstance(so, Mapping): + return so + return None + + +def _validate_against_schema( + payload: Mapping[str, Any], schema: Mapping[str, Any] +) -> Optional[str]: + """Tiny shape validator covering the subset we actually need. + + We deliberately do not pull in `jsonschema` as a test dep: the + matrix's success signal is "does the proxy let the synthetic + StructuredOutput tool round-trip end-to-end", and that's + answerable with a presence + type check over `required` keys. + Any malformed schema beyond that would be a Claude Code bug, + not a LiteLLM-proxy bug, so a deeper check would only add false + failures on the wrong axis. + """ + type_map = { + "integer": int, + "number": (int, float), + "string": str, + "boolean": bool, + "array": list, + "object": Mapping, + } + required = schema.get("required") or [] + properties = schema.get("properties") or {} + for key in required: + if key not in payload: + return f"missing required key {key!r}" + expected = (properties.get(key) or {}).get("type") + if expected and expected in type_map: + if not isinstance(payload[key], type_map[expected]): + return ( + f"key {key!r} has wrong type: " + f"expected {expected}, got {type(payload[key]).__name__}" + ) + # bool is a subclass of int in Python; reject `True`/`False` + # when the schema asked for an integer/number. + if expected in ("integer", "number") and isinstance(payload[key], bool): + return f"key {key!r} is a bool but schema asked for {expected}" + return None + + +def test_structured_outputs_vertex_ai(compat_result): + """Drive `claude --json-schema ...` against the LiteLLM proxy and + assert the trailing `result` event contains a schema-conforming + `structured_output`.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt=PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=["--json-schema", SCHEMA_JSON], + ) + + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + payload = _extract_structured_output(outcome.events) + if payload is None: + error = ( + f"[{model}] no `structured_output` in trailing result event; " + "Claude Code's StructuredOutput tool round-trip did not " + "complete end-to-end through the proxy" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + shape_error = _validate_against_schema(payload, SCHEMA) + if shape_error is not None: + error = f"[{model}] structured_output shape error: {shape_error}; payload={payload}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/test_config.yaml b/tests/e2e/claude_code/test_config.yaml new file mode 100644 index 00000000000..eec68d11dcf --- /dev/null +++ b/tests/e2e/claude_code/test_config.yaml @@ -0,0 +1,103 @@ +# Proxy routing config for the Claude Code Compatibility Matrix PR gate. +# +# The tests under `tests/e2e/claude_code/` only know **alias** names (e.g. +# `claude-haiku-4-5-bedrock-invoke`). The proxy is the layer that maps +# each alias to a real upstream model id, region, and credentials. +# +# Adding a new (feature, provider) cell is a three-step change in the +# test repo (manifest + test file + alias here); changing which upstream +# model a cell exercises is a one-step change here, with no test edits. +# +# Aliases: +# - claude-{tier} → Anthropic API +# - claude-{tier}-bedrock-invoke → Bedrock InvokeModel API +# - claude-{tier}-bedrock-converse → Bedrock Converse API +# - claude-{tier}-vertex → GCP Vertex AI +# - claude-{tier}-azure → Microsoft Foundry (Anthropic deployments) + +model_list: + # ---- Anthropic ---- + - model_name: claude-haiku-4-5 + litellm_params: + model: anthropic/claude-haiku-4-5 + api_key: os.environ/ANTHROPIC_API_KEY + - model_name: claude-sonnet-4-6 + litellm_params: + model: anthropic/claude-sonnet-4-6 + api_key: os.environ/ANTHROPIC_API_KEY + - model_name: claude-opus-4-7 + litellm_params: + model: anthropic/claude-opus-4-7 + api_key: os.environ/ANTHROPIC_API_KEY + + # ---- Bedrock (InvokeModel) ---- + - model_name: claude-haiku-4-5-bedrock-invoke + litellm_params: + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 + aws_region_name: us-east-1 + - model_name: claude-sonnet-4-6-bedrock-invoke + litellm_params: + model: bedrock/us.anthropic.claude-sonnet-4-6 + aws_region_name: us-east-1 + - model_name: claude-opus-4-7-bedrock-invoke + litellm_params: + model: bedrock/us.anthropic.claude-opus-4-7 + aws_region_name: us-east-1 + + # ---- Bedrock (Converse) ---- + - model_name: claude-haiku-4-5-bedrock-converse + litellm_params: + model: bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0 + aws_region_name: us-east-1 + - model_name: claude-sonnet-4-6-bedrock-converse + litellm_params: + model: bedrock/converse/us.anthropic.claude-sonnet-4-6 + aws_region_name: us-east-1 + - model_name: claude-opus-4-7-bedrock-converse + litellm_params: + model: bedrock/converse/us.anthropic.claude-opus-4-7 + aws_region_name: us-east-1 + + # ---- Vertex AI ---- + - model_name: claude-haiku-4-5-vertex + litellm_params: + model: vertex_ai/claude-haiku-4-5 + vertex_ai_project: os.environ/VERTEXAI_PROJECT + vertex_ai_location: os.environ/VERTEXAI_LOCATION + - model_name: claude-sonnet-4-6-vertex + litellm_params: + model: vertex_ai/claude-sonnet-4-6 + vertex_ai_project: os.environ/VERTEXAI_PROJECT + vertex_ai_location: os.environ/VERTEXAI_LOCATION + - model_name: claude-opus-4-7-vertex + litellm_params: + model: vertex_ai/claude-opus-4-7 + vertex_ai_project: os.environ/VERTEXAI_PROJECT + vertex_ai_location: os.environ/VERTEXAI_LOCATION + + # ---- Microsoft Foundry (Anthropic deployments on Azure) ---- + - model_name: claude-haiku-4-5-azure + litellm_params: + model: azure_ai/claude-haiku-4-5 + api_base: os.environ/AZURE_FOUNDRY_API_BASE + api_key: os.environ/AZURE_FOUNDRY_API_KEY + - model_name: claude-sonnet-4-6-azure + litellm_params: + model: azure_ai/claude-sonnet-4-6 + api_base: os.environ/AZURE_FOUNDRY_API_BASE + api_key: os.environ/AZURE_FOUNDRY_API_KEY + - model_name: claude-opus-4-7-azure + litellm_params: + model: azure_ai/claude-opus-4-7 + api_base: os.environ/AZURE_FOUNDRY_API_BASE + api_key: os.environ/AZURE_FOUNDRY_API_KEY + +general_settings: + # Claude Code sends provider-specific headers (e.g. anthropic-beta) we + # want to forward verbatim to the upstream so the wire-shape under + # test matches what real customers send. + forward_client_headers_to_llm_api: true + +litellm_settings: + drop_params: true + modify_params: true diff --git a/tests/e2e/claude_code/thinking/__init__.py b/tests/e2e/claude_code/thinking/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/thinking/test_anthropic.py b/tests/e2e/claude_code/thinking/test_anthropic.py new file mode 100644 index 00000000000..1090d1b384e --- /dev/null +++ b/tests/e2e/claude_code/thinking/test_anthropic.py @@ -0,0 +1,132 @@ +"""thinking x Anthropic. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Anthropic, enable extended thinking via `--effort high`, and assert +that the upstream returned a `thinking` content block. This proves the +proxy preserves Anthropic's `thinking` request parameter and the +upstream response's `thinking` content blocks end-to-end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/thinking/test_anthropic.py + ^^^^^^^^ ^^^^^^^^^ + feature_id provider + +The three Claude tiers run in parallel inside this single test, with +one `compat_result.add(...)` entry per model so the matrix builder +still sees three rows for this (feature, provider). +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +# --effort max maps to the largest thinking budget on every supported +# Claude tier; the test cares about wire shape, not answer quality. We +# use a CLI flag (rather than the legacy MAX_THINKING_TOKENS env var) +# because Claude Code 2.x reads thinking config from --effort, not from +# the env, and silently no-ops the env var. We use `max` rather than +# `high` because Sonnet 4.6 / Opus 4.7 only emit thinking blocks when +# the budget is generous and the prompt is non-trivial. +THINKING_ARGS = ["--effort", "max"] +# A puzzle non-trivial enough that Sonnet/Opus actually engage thinking +# rather than answer from memory. Trivial arithmetic ("3-2=?") is +# optimized away on the modern tiers and arrives without a thinking +# block, which would make this test silently false-fail under +# `--effort max`. Haiku 4.5 thinks even for trivial prompts; Sonnet 4.6 +# and Opus 4.7 only emit thinking when the upstream judges it useful. +THINKING_PROMPT = ( + "I have a 3-gallon jug and a 5-gallon jug. How can I measure " + "exactly 4 gallons of water? Think through the steps carefully." +) + + +def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: + """Walk the stream-json events and return True if any assistant + message included a `thinking` content block.""" + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "thinking": + return True + return False + + +def test_thinking_anthropic(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy with thinking + enabled and assert a `thinking` content block was emitted.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt=THINKING_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=THINKING_ARGS, + ) + + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_thinking_block(outcome.events): + error = ( + f"[{model}] no `thinking` content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/thinking/test_azure.py b/tests/e2e/claude_code/thinking/test_azure.py new file mode 100644 index 00000000000..1fd5138d574 --- /dev/null +++ b/tests/e2e/claude_code/thinking/test_azure.py @@ -0,0 +1,120 @@ +"""thinking x Azure (Microsoft Foundry). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Anthropic's models hosted in Microsoft Foundry on +Azure, enable extended thinking via `--effort high`, and assert +that the upstream returned a `thinking` content block. + +Foundry's Claude deployments advertise `supports_reasoning: true` in +LiteLLM's pricing metadata; the `thinking={"type": "enabled", ...}` +parameter passes through `azure_ai/claude-*` to Foundry's +`/anthropic/v1/messages` endpoint unchanged. Note that +`claude-opus-4-7-preview` documents thinking as not supported on +Foundry; if that lands, this row may flip to a partial pass and we'll +re-evaluate. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/thinking/test_azure.py + ^^^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + +THINKING_ARGS = ["--effort", "max"] +THINKING_PROMPT = ( + "I have a 3-gallon jug and a 5-gallon jug. How can I measure " + "exactly 4 gallons of water? Think through the steps carefully." +) + + +def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "thinking": + return True + return False + + +def test_thinking_azure(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy with thinking + enabled and assert a `thinking` content block was emitted.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt=THINKING_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=THINKING_ARGS, + ) + + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_thinking_block(outcome.events): + error = ( + f"[{model}] no `thinking` content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/thinking/test_bedrock_converse.py b/tests/e2e/claude_code/thinking/test_bedrock_converse.py new file mode 100644 index 00000000000..793ce8542da --- /dev/null +++ b/tests/e2e/claude_code/thinking/test_bedrock_converse.py @@ -0,0 +1,112 @@ +"""thinking x Bedrock (Converse). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the unified `Converse` API path, +enable extended thinking via `--effort high`, and assert that the +upstream returned a `thinking` content block. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/thinking/test_bedrock_converse.py + ^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + +THINKING_ARGS = ["--effort", "max"] +THINKING_PROMPT = ( + "I have a 3-gallon jug and a 5-gallon jug. How can I measure " + "exactly 4 gallons of water? Think through the steps carefully." +) + + +def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "thinking": + return True + return False + + +def test_thinking_bedrock_converse(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy with thinking + enabled and assert a `thinking` content block was emitted.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt=THINKING_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=THINKING_ARGS, + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_thinking_block(outcome.events): + error = ( + f"[{model}] no `thinking` content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/thinking/test_bedrock_invoke.py b/tests/e2e/claude_code/thinking/test_bedrock_invoke.py new file mode 100644 index 00000000000..e31b60eb004 --- /dev/null +++ b/tests/e2e/claude_code/thinking/test_bedrock_invoke.py @@ -0,0 +1,112 @@ +"""thinking x Bedrock (Invoke). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the legacy `InvokeModel` API path, +enable extended thinking via `--effort high`, and assert that the +upstream returned a `thinking` content block. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/thinking/test_bedrock_invoke.py + ^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + +THINKING_ARGS = ["--effort", "max"] +THINKING_PROMPT = ( + "I have a 3-gallon jug and a 5-gallon jug. How can I measure " + "exactly 4 gallons of water? Think through the steps carefully." +) + + +def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "thinking": + return True + return False + + +def test_thinking_bedrock_invoke(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy with thinking + enabled and assert a `thinking` content block was emitted.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt=THINKING_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=THINKING_ARGS, + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_thinking_block(outcome.events): + error = ( + f"[{model}] no `thinking` content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/thinking/test_vertex_ai.py b/tests/e2e/claude_code/thinking/test_vertex_ai.py new file mode 100644 index 00000000000..c5c7df1f9b8 --- /dev/null +++ b/tests/e2e/claude_code/thinking/test_vertex_ai.py @@ -0,0 +1,112 @@ +"""thinking x Vertex AI. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Anthropic's models on Google Cloud Vertex AI, enable +extended thinking via `--effort high`, and assert that the +upstream returned a `thinking` content block. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/thinking/test_vertex_ai.py + ^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + +THINKING_ARGS = ["--effort", "max"] +THINKING_PROMPT = ( + "I have a 3-gallon jug and a 5-gallon jug. How can I measure " + "exactly 4 gallons of water? Think through the steps carefully." +) + + +def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "thinking": + return True + return False + + +def test_thinking_vertex_ai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy with thinking + enabled and assert a `thinking` content block was emitted.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt=THINKING_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=THINKING_ARGS, + ) + + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_thinking_block(outcome.events): + error = ( + f"[{model}] no `thinking` content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/thinking_with_tool_use/__init__.py b/tests/e2e/claude_code/thinking_with_tool_use/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py b/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py new file mode 100644 index 00000000000..2c573ea039e --- /dev/null +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py @@ -0,0 +1,158 @@ +"""thinking_with_tool_use x Anthropic. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Anthropic, enable extended thinking via `--effort high`, allow +the built-in `Bash` tool, and ask Claude to plan-and-execute a task +that requires both reasoning and a tool call. Assert the upstream +returned both a `thinking` content block and a `tool_use` content +block in the same turn — proving the proxy preserves the wire shape +where extended thinking and tool use coexist. + +This is the cell that historically catches the most provider bugs: +"thinking blocks cannot be modified" 400s, the recurring Bedrock +"thinking.type.enabled is not supported" error, and the +`fine-grained-tool-streaming` + `interleaved-thinking` beta-header +interactions. A regression in any of those collapses this cell. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py + ^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +# Extended thinking on, with a small budget — enough to surface a +# non-empty thinking block on a trivial reasoning prompt without +# blowing up wall time. +THINKING_ARGS = ["--effort", "max"] + +# Prompt designed to force both blocks: the model has to *reason* about +# what command to run before *invoking* the Bash tool. Using a fixed +# expected output keeps the assertion focused on the wire shape rather +# than on answer quality. +# Prompt fixes the exact bash command to `echo pong`. The thinking +# block is preserved (the model reasons about why `echo pong` works), +# but the executed command is pinned so the cell can run under the +# tight `Bash(echo pong) + dontAsk` permission below — see +# `tool_use/test_anthropic.py` for the full security rationale. +THINKING_TOOL_PROMPT = ( + "Think step by step about why the command `echo pong` prints just the " + "word 'pong'. Then use the Bash tool to run exactly the command " + "`echo pong` and report what it printed." +) +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] + + +def _has_block_type( + events: Sequence[Mapping[str, Any]], + block_type: str, +) -> bool: + """Walk the stream-json events and return True if any assistant + message included a content block of the given type.""" + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == block_type: + return True + return False + + +def test_thinking_with_tool_use_anthropic(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy with thinking + enabled and tool use, and assert both `thinking` and `tool_use` + content blocks landed in the same turn.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt=THINKING_TOOL_PROMPT, + base_url=base_url, + api_key=api_key, + # thinking + tools combined into a single extra_args; see THINKING_ARGS + extra_args=THINKING_ARGS + TOOL_USE_ARGS, + ) + + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "thinking"): + error = ( + f"[{model}] no `thinking` content block observed; thinking " + f"either disabled by the proxy or stripped by the upstream" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "tool_use"): + error = ( + f"[{model}] no `tool_use` content block observed alongside " + f"thinking; the proxy may have dropped tools when thinking is on" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py b/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py new file mode 100644 index 00000000000..3d65e82cdec --- /dev/null +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py @@ -0,0 +1,130 @@ +"""thinking_with_tool_use x Microsoft Foundry (Azure). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Microsoft Foundry's Anthropic deployments on Azure, +enable extended thinking via `--effort high`, allow the built-in +`Bash` tool, and ask Claude to plan-and-execute a task that requires +both reasoning and a tool call. Assert the upstream returned both a +`thinking` content block and a `tool_use` content block in the same +turn. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/thinking_with_tool_use/test_azure.py + ^^^^^^^^^^^^^^^^^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + +THINKING_ARGS = ["--effort", "max"] +# Prompt + Bash restriction pin the executed command to `echo pong`; +# see `tool_use/test_anthropic.py` for the security rationale. +THINKING_TOOL_PROMPT = ( + "Think step by step about why the command `echo pong` prints just the " + "word 'pong'. Then use the Bash tool to run exactly the command " + "`echo pong` and report what it printed." +) +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] + + +def _has_block_type( + events: Sequence[Mapping[str, Any]], + block_type: str, +) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == block_type: + return True + return False + + +def test_thinking_with_tool_use_azure(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt=THINKING_TOOL_PROMPT, + base_url=base_url, + api_key=api_key, + # thinking + tools combined into a single extra_args; see THINKING_ARGS + extra_args=THINKING_ARGS + TOOL_USE_ARGS, + ) + + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "thinking"): + error = f"[{model}] no `thinking` content block observed" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "tool_use"): + error = f"[{model}] no `tool_use` content block observed alongside thinking" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py new file mode 100644 index 00000000000..eb916323546 --- /dev/null +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py @@ -0,0 +1,135 @@ +"""thinking_with_tool_use x Bedrock (Converse). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the `Converse` API, enable extended +thinking via `--effort high`, allow the built-in `Bash` tool, and +ask Claude to plan-and-execute a task that requires both reasoning and +a tool call. Assert the upstream returned both a `thinking` content +block and a `tool_use` content block in the same turn. + +The Converse API has its own `additionalModelRequestFields.thinking` +shape and its own tool-use envelope; this cell catches gateway +regressions where the proxy fails to translate between Anthropic's +`thinking` parameter and Converse's reasoning configuration when tools +are also present. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py + ^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + +THINKING_ARGS = ["--effort", "max"] +# Prompt + Bash restriction pin the executed command to `echo pong`; +# see `tool_use/test_anthropic.py` for the security rationale. +THINKING_TOOL_PROMPT = ( + "Think step by step about why the command `echo pong` prints just the " + "word 'pong'. Then use the Bash tool to run exactly the command " + "`echo pong` and report what it printed." +) +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] + + +def _has_block_type( + events: Sequence[Mapping[str, Any]], + block_type: str, +) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == block_type: + return True + return False + + +def test_thinking_with_tool_use_bedrock_converse(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt=THINKING_TOOL_PROMPT, + base_url=base_url, + api_key=api_key, + # thinking + tools combined into a single extra_args; see THINKING_ARGS + extra_args=THINKING_ARGS + TOOL_USE_ARGS, + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "thinking"): + error = f"[{model}] no `thinking` content block observed" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "tool_use"): + error = f"[{model}] no `tool_use` content block observed alongside thinking" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py new file mode 100644 index 00000000000..d1a61a59772 --- /dev/null +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py @@ -0,0 +1,137 @@ +"""thinking_with_tool_use x Bedrock (Invoke). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the legacy `InvokeModel` API path, +enable extended thinking via `--effort high`, allow the built-in +`Bash` tool, and ask Claude to plan-and-execute a task that requires +both reasoning and a tool call. Assert the upstream returned both a +`thinking` content block and a `tool_use` content block in the same +turn. + +This is the cell most likely to flush out Bedrock-specific bugs in +LiteLLM's Anthropic <-> Bedrock translation: the recurring +"thinking.type.enabled is not supported" 400 error has reappeared on +several Bedrock model routes (notably application inference profile +ARNs), and the only reliable signal that the fix is wired through the +proxy is a successful round-trip on this cell. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py + ^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + +THINKING_ARGS = ["--effort", "max"] +# Prompt + Bash restriction pin the executed command to `echo pong`; +# see `tool_use/test_anthropic.py` for the security rationale. +THINKING_TOOL_PROMPT = ( + "Think step by step about why the command `echo pong` prints just the " + "word 'pong'. Then use the Bash tool to run exactly the command " + "`echo pong` and report what it printed." +) +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] + + +def _has_block_type( + events: Sequence[Mapping[str, Any]], + block_type: str, +) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == block_type: + return True + return False + + +def test_thinking_with_tool_use_bedrock_invoke(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt=THINKING_TOOL_PROMPT, + base_url=base_url, + api_key=api_key, + # thinking + tools combined into a single extra_args; see THINKING_ARGS + extra_args=THINKING_ARGS + TOOL_USE_ARGS, + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "thinking"): + error = f"[{model}] no `thinking` content block observed" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "tool_use"): + error = f"[{model}] no `tool_use` content block observed alongside thinking" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py b/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py new file mode 100644 index 00000000000..285419c67f7 --- /dev/null +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py @@ -0,0 +1,135 @@ +"""thinking_with_tool_use x Vertex AI. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to GCP Vertex AI, enable extended thinking via +`--effort high`, allow the built-in `Bash` tool, and ask Claude +to plan-and-execute a task that requires both reasoning and a tool +call. Assert the upstream returned both a `thinking` content block and +a `tool_use` content block in the same turn. + +Vertex AI exposes Anthropic models via `:rawPredict` / +`:streamRawPredict` and has its own beta-header allowlist. This cell +catches gateway regressions where the proxy strips +`anthropic-beta: interleaved-thinking-2025-05-14` (or the equivalent +header set the upstream needs) on the way to Vertex. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py + ^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + +THINKING_ARGS = ["--effort", "max"] +# Prompt + Bash restriction pin the executed command to `echo pong`; +# see `tool_use/test_anthropic.py` for the security rationale. +THINKING_TOOL_PROMPT = ( + "Think step by step about why the command `echo pong` prints just the " + "word 'pong'. Then use the Bash tool to run exactly the command " + "`echo pong` and report what it printed." +) +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] + + +def _has_block_type( + events: Sequence[Mapping[str, Any]], + block_type: str, +) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == block_type: + return True + return False + + +def test_thinking_with_tool_use_vertex_ai(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt=THINKING_TOOL_PROMPT, + base_url=base_url, + api_key=api_key, + # thinking + tools combined into a single extra_args; see THINKING_ARGS + extra_args=THINKING_ARGS + TOOL_USE_ARGS, + ) + + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "thinking"): + error = f"[{model}] no `thinking` content block observed" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "tool_use"): + error = f"[{model}] no `tool_use` content block observed alongside thinking" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_search/__init__.py b/tests/e2e/claude_code/tool_search/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/tool_search/test_anthropic.py b/tests/e2e/claude_code/tool_search/test_anthropic.py new file mode 100644 index 00000000000..3495c882e06 --- /dev/null +++ b/tests/e2e/claude_code/tool_search/test_anthropic.py @@ -0,0 +1,99 @@ +"""tool_search x Anthropic. + +HTTP-probe row. Sends a single `/v1/messages` request whose `tools` +array includes a `tool_search_tool_regex_20251119` discovery tool, and +asserts the proxy round-trips it to the upstream without a 400. This +verifies LiteLLM's tool-search beta-header translation +(`advanced-tool-use-2025-11-20` for Anthropic-shape providers, +`tool-search-tool-2025-10-19` for Vertex/Bedrock) survives end-to-end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_search/test_anthropic.py + ^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider + +Why HTTP probe instead of CLI / MCP fan-out: + +Real Claude Code activates tool_search by registering >N MCP tools and +relying on the model's internal heuristic to call the discovery tool +before any user tool. That setup requires standing up a stub MCP +server that exposes 50+ tool stubs and depends on Claude Code's +auto-deferral heuristic continuing to fire at today's tool count -- +both of which break silently when Claude Code's threshold changes +between releases. + +The bugs LiteLLM has actually shipped fixes for in this area +(2.1.117, 2.1.72, 2.1.70 in the Claude Code release notes) are +beta-header translation and proxy-side type recognition, not MCP +fan-out behavior. An HTTP probe hits exactly that surface: the +request goes out with a `tool_search_tool_regex_20251119` tool type, +the proxy is responsible for attaching the per-provider beta header +and forwarding, and the upstream either accepts or 400s. A red cell +here is always a proxy-side regression, not a flaky model-behavior +artifact. + +Three Claude tiers are probed in sequence (count is too low to be +worth the parallelism overhead, and HTTP probes don't compete for +the proxy's `--num-workers` slots the way CLI subprocess runs do). +The matrix's "all three must pass" rule still applies via the +per-cell aggregator. +""" + +from __future__ import annotations + +import os + +import pytest + +from claude_code.http_probe import ( + assert_tool_search_shape, + probe_tool_search, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + + +def test_tool_search_anthropic(compat_result): + """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` + tool and assert the proxy + upstream accept it for every Anthropic + tier.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + failures = [] + for model in ANTHROPIC_MODELS: + result = probe_tool_search(base_url=base_url, api_key=api_key, model=model) + shape_error = assert_tool_search_shape(result) + if shape_error is not None: + error = f"[{model}] tool_search probe failed: {shape_error}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_search/test_azure.py b/tests/e2e/claude_code/tool_search/test_azure.py new file mode 100644 index 00000000000..1d9cb5673c5 --- /dev/null +++ b/tests/e2e/claude_code/tool_search/test_azure.py @@ -0,0 +1,99 @@ +"""tool_search x Azure (Microsoft Foundry). + +HTTP-probe row. Sends a single `/v1/messages` request whose `tools` +array includes a `tool_search_tool_regex_20251119` discovery tool, and +asserts the proxy round-trips it to the upstream without a 400. This +verifies LiteLLM's tool-search beta-header translation +(`advanced-tool-use-2025-11-20` for Anthropic-shape providers, +`tool-search-tool-2025-10-19` for Vertex/Bedrock) survives end-to-end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_search/test_azure.py + ^^^^^^^^^^^ ^^^^^ + feature_id provider + +Why HTTP probe instead of CLI / MCP fan-out: + +Real Claude Code activates tool_search by registering >N MCP tools and +relying on the model's internal heuristic to call the discovery tool +before any user tool. That setup requires standing up a stub MCP +server that exposes 50+ tool stubs and depends on Claude Code's +auto-deferral heuristic continuing to fire at today's tool count -- +both of which break silently when Claude Code's threshold changes +between releases. + +The bugs LiteLLM has actually shipped fixes for in this area +(2.1.117, 2.1.72, 2.1.70 in the Claude Code release notes) are +beta-header translation and proxy-side type recognition, not MCP +fan-out behavior. An HTTP probe hits exactly that surface: the +request goes out with a `tool_search_tool_regex_20251119` tool type, +the proxy is responsible for attaching the per-provider beta header +and forwarding, and the upstream either accepts or 400s. A red cell +here is always a proxy-side regression, not a flaky model-behavior +artifact. + +Three Claude tiers are probed in sequence (count is too low to be +worth the parallelism overhead, and HTTP probes don't compete for +the proxy's `--num-workers` slots the way CLI subprocess runs do). +The matrix's "all three must pass" rule still applies via the +per-cell aggregator. +""" + +from __future__ import annotations + +import os + +import pytest + +from claude_code.http_probe import ( + assert_tool_search_shape, + probe_tool_search, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + + +def test_tool_search_azure(compat_result): + """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` + tool and assert the proxy + upstream accept it for every Azure (Microsoft Foundry) + tier.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + failures = [] + for model in AZURE_MODELS: + result = probe_tool_search(base_url=base_url, api_key=api_key, model=model) + shape_error = assert_tool_search_shape(result) + if shape_error is not None: + error = f"[{model}] tool_search probe failed: {shape_error}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_search/test_bedrock_converse.py b/tests/e2e/claude_code/tool_search/test_bedrock_converse.py new file mode 100644 index 00000000000..5ca0792529a --- /dev/null +++ b/tests/e2e/claude_code/tool_search/test_bedrock_converse.py @@ -0,0 +1,99 @@ +"""tool_search x Bedrock (Converse). + +HTTP-probe row. Sends a single `/v1/messages` request whose `tools` +array includes a `tool_search_tool_regex_20251119` discovery tool, and +asserts the proxy round-trips it to the upstream without a 400. This +verifies LiteLLM's tool-search beta-header translation +(`advanced-tool-use-2025-11-20` for Anthropic-shape providers, +`tool-search-tool-2025-10-19` for Vertex/Bedrock) survives end-to-end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_search/test_bedrock_converse.py + ^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider + +Why HTTP probe instead of CLI / MCP fan-out: + +Real Claude Code activates tool_search by registering >N MCP tools and +relying on the model's internal heuristic to call the discovery tool +before any user tool. That setup requires standing up a stub MCP +server that exposes 50+ tool stubs and depends on Claude Code's +auto-deferral heuristic continuing to fire at today's tool count -- +both of which break silently when Claude Code's threshold changes +between releases. + +The bugs LiteLLM has actually shipped fixes for in this area +(2.1.117, 2.1.72, 2.1.70 in the Claude Code release notes) are +beta-header translation and proxy-side type recognition, not MCP +fan-out behavior. An HTTP probe hits exactly that surface: the +request goes out with a `tool_search_tool_regex_20251119` tool type, +the proxy is responsible for attaching the per-provider beta header +and forwarding, and the upstream either accepts or 400s. A red cell +here is always a proxy-side regression, not a flaky model-behavior +artifact. + +Three Claude tiers are probed in sequence (count is too low to be +worth the parallelism overhead, and HTTP probes don't compete for +the proxy's `--num-workers` slots the way CLI subprocess runs do). +The matrix's "all three must pass" rule still applies via the +per-cell aggregator. +""" + +from __future__ import annotations + +import os + +import pytest + +from claude_code.http_probe import ( + assert_tool_search_shape, + probe_tool_search, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + + +def test_tool_search_bedrock_converse(compat_result): + """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` + tool and assert the proxy + upstream accept it for every Bedrock (Converse) + tier.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + result = probe_tool_search(base_url=base_url, api_key=api_key, model=model) + shape_error = assert_tool_search_shape(result) + if shape_error is not None: + error = f"[{model}] tool_search probe failed: {shape_error}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py new file mode 100644 index 00000000000..21bb33e34bd --- /dev/null +++ b/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py @@ -0,0 +1,99 @@ +"""tool_search x Bedrock (Invoke). + +HTTP-probe row. Sends a single `/v1/messages` request whose `tools` +array includes a `tool_search_tool_regex_20251119` discovery tool, and +asserts the proxy round-trips it to the upstream without a 400. This +verifies LiteLLM's tool-search beta-header translation +(`advanced-tool-use-2025-11-20` for Anthropic-shape providers, +`tool-search-tool-2025-10-19` for Vertex/Bedrock) survives end-to-end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_search/test_bedrock_invoke.py + ^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider + +Why HTTP probe instead of CLI / MCP fan-out: + +Real Claude Code activates tool_search by registering >N MCP tools and +relying on the model's internal heuristic to call the discovery tool +before any user tool. That setup requires standing up a stub MCP +server that exposes 50+ tool stubs and depends on Claude Code's +auto-deferral heuristic continuing to fire at today's tool count -- +both of which break silently when Claude Code's threshold changes +between releases. + +The bugs LiteLLM has actually shipped fixes for in this area +(2.1.117, 2.1.72, 2.1.70 in the Claude Code release notes) are +beta-header translation and proxy-side type recognition, not MCP +fan-out behavior. An HTTP probe hits exactly that surface: the +request goes out with a `tool_search_tool_regex_20251119` tool type, +the proxy is responsible for attaching the per-provider beta header +and forwarding, and the upstream either accepts or 400s. A red cell +here is always a proxy-side regression, not a flaky model-behavior +artifact. + +Three Claude tiers are probed in sequence (count is too low to be +worth the parallelism overhead, and HTTP probes don't compete for +the proxy's `--num-workers` slots the way CLI subprocess runs do). +The matrix's "all three must pass" rule still applies via the +per-cell aggregator. +""" + +from __future__ import annotations + +import os + +import pytest + +from claude_code.http_probe import ( + assert_tool_search_shape, + probe_tool_search, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + + +def test_tool_search_bedrock_invoke(compat_result): + """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` + tool and assert the proxy + upstream accept it for every Bedrock (Invoke) + tier.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + result = probe_tool_search(base_url=base_url, api_key=api_key, model=model) + shape_error = assert_tool_search_shape(result) + if shape_error is not None: + error = f"[{model}] tool_search probe failed: {shape_error}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_search/test_vertex_ai.py b/tests/e2e/claude_code/tool_search/test_vertex_ai.py new file mode 100644 index 00000000000..f91400b1817 --- /dev/null +++ b/tests/e2e/claude_code/tool_search/test_vertex_ai.py @@ -0,0 +1,99 @@ +"""tool_search x Vertex AI. + +HTTP-probe row. Sends a single `/v1/messages` request whose `tools` +array includes a `tool_search_tool_regex_20251119` discovery tool, and +asserts the proxy round-trips it to the upstream without a 400. This +verifies LiteLLM's tool-search beta-header translation +(`advanced-tool-use-2025-11-20` for Anthropic-shape providers, +`tool-search-tool-2025-10-19` for Vertex/Bedrock) survives end-to-end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_search/test_vertex_ai.py + ^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider + +Why HTTP probe instead of CLI / MCP fan-out: + +Real Claude Code activates tool_search by registering >N MCP tools and +relying on the model's internal heuristic to call the discovery tool +before any user tool. That setup requires standing up a stub MCP +server that exposes 50+ tool stubs and depends on Claude Code's +auto-deferral heuristic continuing to fire at today's tool count -- +both of which break silently when Claude Code's threshold changes +between releases. + +The bugs LiteLLM has actually shipped fixes for in this area +(2.1.117, 2.1.72, 2.1.70 in the Claude Code release notes) are +beta-header translation and proxy-side type recognition, not MCP +fan-out behavior. An HTTP probe hits exactly that surface: the +request goes out with a `tool_search_tool_regex_20251119` tool type, +the proxy is responsible for attaching the per-provider beta header +and forwarding, and the upstream either accepts or 400s. A red cell +here is always a proxy-side regression, not a flaky model-behavior +artifact. + +Three Claude tiers are probed in sequence (count is too low to be +worth the parallelism overhead, and HTTP probes don't compete for +the proxy's `--num-workers` slots the way CLI subprocess runs do). +The matrix's "all three must pass" rule still applies via the +per-cell aggregator. +""" + +from __future__ import annotations + +import os + +import pytest + +from claude_code.http_probe import ( + assert_tool_search_shape, + probe_tool_search, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + + +def test_tool_search_vertex_ai(compat_result): + """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` + tool and assert the proxy + upstream accept it for every Vertex AI + tier.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", + pytrace=False, + ) + + failures = [] + for model in VERTEX_AI_MODELS: + result = probe_tool_search(base_url=base_url, api_key=api_key, model=model) + shape_error = assert_tool_search_shape(result) + if shape_error is not None: + error = f"[{model}] tool_search probe failed: {shape_error}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use/__init__.py b/tests/e2e/claude_code/tool_use/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/tool_use/test_anthropic.py b/tests/e2e/claude_code/tool_use/test_anthropic.py new file mode 100644 index 00000000000..7d2aa4be683 --- /dev/null +++ b/tests/e2e/claude_code/tool_use/test_anthropic.py @@ -0,0 +1,129 @@ +"""tool_use x Anthropic. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Anthropic, ask Claude to invoke a built-in tool (`Bash`), and assert +that the upstream returned a `tool_use` content block. This proves the +proxy preserves Claude Code's tool-call wire shape end-to-end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use/test_anthropic.py + ^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +# Built-in tool-use prompt: ask Claude to use the `Bash` tool. The CLI +# allow-lists the tool via `--allowed-tools` so the run completes without +# an interactive permission prompt. +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +# Restrict the Bash tool to the exact command `echo pong` and put the +# CLI in `dontAsk` mode so anything else the model returns is auto- +# denied instead of executed. `dontAsk` mode in headless `--print` mode +# only runs tools matching an explicit `allow` rule (plus the built-in +# read-only set), so a compromised provider response cannot turn the +# `Bash` allowlist into arbitrary host execution (which would expose +# `docker inspect compat-proxy` / `/proc//environ` and +# thereby provider credentials living in the proxy container). +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + """Walk the stream-json events and return True if any assistant + message included a `tool_use` content block.""" + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def test_tool_use_anthropic(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + tool call was emitted on the wire.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use/test_azure.py b/tests/e2e/claude_code/tool_use/test_azure.py new file mode 100644 index 00000000000..484f50a5508 --- /dev/null +++ b/tests/e2e/claude_code/tool_use/test_azure.py @@ -0,0 +1,123 @@ +"""tool_use x Azure (Microsoft Foundry). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Anthropic's models hosted in Microsoft Foundry on +Azure, ask Claude to invoke a built-in tool (`Bash`), and assert that +the upstream returned a `tool_use` content block. + +Foundry's Anthropic deployments support function/tool calling +identically to anthropic.com; LiteLLM's `azure_ai/claude-*` route +inherits the full Anthropic tool-use transformation. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use/test_azure.py + ^^^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def test_tool_use_azure(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + tool call was emitted on the wire.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use/test_bedrock_converse.py b/tests/e2e/claude_code/tool_use/test_bedrock_converse.py new file mode 100644 index 00000000000..7d1b58fce90 --- /dev/null +++ b/tests/e2e/claude_code/tool_use/test_bedrock_converse.py @@ -0,0 +1,119 @@ +"""tool_use x Bedrock (Converse). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the unified `Converse` API path, ask +Claude to invoke a built-in tool (`Bash`), and assert that the upstream +returned a `tool_use` content block. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use/test_bedrock_converse.py + ^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def test_tool_use_bedrock_converse(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + tool call was emitted on the wire.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py new file mode 100644 index 00000000000..7d2b72b951d --- /dev/null +++ b/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py @@ -0,0 +1,119 @@ +"""tool_use x Bedrock (Invoke). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the legacy `InvokeModel` API path, +ask Claude to invoke a built-in tool (`Bash`), and assert that the +upstream returned a `tool_use` content block. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use/test_bedrock_invoke.py + ^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def test_tool_use_bedrock_invoke(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + tool call was emitted on the wire.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use/test_vertex_ai.py b/tests/e2e/claude_code/tool_use/test_vertex_ai.py new file mode 100644 index 00000000000..0a8ecc9f7a7 --- /dev/null +++ b/tests/e2e/claude_code/tool_use/test_vertex_ai.py @@ -0,0 +1,119 @@ +"""tool_use x Vertex AI. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Anthropic's models on Google Cloud Vertex AI, ask +Claude to invoke a built-in tool (`Bash`), and assert that the upstream +returned a `tool_use` content block. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use/test_vertex_ai.py + ^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def test_tool_use_vertex_ai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + tool call was emitted on the wire.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use_streaming/__init__.py b/tests/e2e/claude_code/tool_use_streaming/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py b/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py new file mode 100644 index 00000000000..9aa94c89241 --- /dev/null +++ b/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py @@ -0,0 +1,165 @@ +"""tool_use_streaming x Anthropic. + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode (with `--include-partial-messages`) against a running LiteLLM +proxy that routes to Anthropic, ask Claude to invoke a built-in tool +(`Bash`), and assert that the upstream (a) emitted a `tool_use` content +block and (b) actually streamed the tool input incrementally — i.e. +`input_json_delta` stream events were observed for the block. + +This is the "fine-grained tool streaming" path. Historically gateways +break it in two ways: they either buffer/collapse the streamed tool +input into a single complete block (no `input_json_delta` records +reach the client) or they strip the +`fine-grained-tool-streaming-2025-05-14` beta header and the upstream +falls back to non-streaming tool_use. Both regressions are caught by +the assertions below. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use_streaming/test_anthropic.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +# Same shape as the non-streaming `tool_use` cell: ask Claude to call +# the built-in `Bash` tool. `--include-partial-messages` surfaces the +# raw SSE records as `stream_event` entries in the stream-json output, +# which is the wire-level signal for whether the proxy preserved +# incremental `input_json_delta` events for the tool_use block. +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", + "--include-partial-messages", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + """Walk the stream-json events and return True if any assistant + message included a `tool_use` content block.""" + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: + """Count `input_json_delta` records among the `stream_event` + entries. Zero means the proxy collapsed the streamed tool input + into a single complete block instead of forwarding the incremental + deltas the upstream emitted.""" + inner_events = ( + event.get("event") for event in events if event.get("type") == "stream_event" + ) + return sum( + 1 + for inner in inner_events + if isinstance(inner, Mapping) + and inner.get("type") == "content_block_delta" + and isinstance(inner.get("delta"), Mapping) + and inner["delta"].get("type") == "input_json_delta" + ) + + +def test_tool_use_streaming_anthropic(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert the + proxy preserves fine-grained tool streaming end-to-end.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _count_input_json_deltas(outcome.events) == 0: + error = ( + f"[{model}] no input_json_delta stream events observed; proxy " + f"likely buffered the tool input into a complete block or " + f"stripped fine-grained tool streaming" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_azure.py b/tests/e2e/claude_code/tool_use_streaming/test_azure.py new file mode 100644 index 00000000000..c73062b72cd --- /dev/null +++ b/tests/e2e/claude_code/tool_use_streaming/test_azure.py @@ -0,0 +1,148 @@ +"""tool_use_streaming x Microsoft Foundry (Azure). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Claude requests to +Microsoft Foundry's Anthropic deployments on Azure, ask Claude to +invoke a built-in tool (`Bash`), and assert that the upstream (a) +emitted a `tool_use` content block and (b) actually streamed events +incrementally. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use_streaming/test_azure.py + ^^^^^^^^^^^^^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", + "--include-partial-messages", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: + """Count `input_json_delta` records among the `stream_event` + entries. Zero means the proxy collapsed the streamed tool input + into a single complete block instead of forwarding the incremental + deltas the upstream emitted.""" + inner_events = ( + event.get("event") for event in events if event.get("type") == "stream_event" + ) + return sum( + 1 + for inner in inner_events + if isinstance(inner, Mapping) + and inner.get("type") == "content_block_delta" + and isinstance(inner.get("delta"), Mapping) + and inner["delta"].get("type") == "input_json_delta" + ) + + +def test_tool_use_streaming_azure(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _count_input_json_deltas(outcome.events) == 0: + error = ( + f"[{model}] no input_json_delta stream events observed; proxy " + f"likely buffered the tool input into a complete block or " + f"stripped fine-grained tool streaming" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py new file mode 100644 index 00000000000..3642551c7c3 --- /dev/null +++ b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py @@ -0,0 +1,154 @@ +"""tool_use_streaming x Bedrock (Converse). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Claude requests to +AWS Bedrock via the `Converse` API (`ConverseStream`), ask Claude to +invoke a built-in tool (`Bash`), and assert that the upstream (a) +emitted a `tool_use` content block and (b) actually streamed events +incrementally. + +Bedrock Converse has its own tool-streaming envelope (`toolUse` blocks +with `delta` chunks); this cell catches gateway regressions where the +proxy buffers the response or fails to translate Converse's streaming +envelope back to the Anthropic `message_*` event shape Claude Code +expects. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", + "--include-partial-messages", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: + """Count `input_json_delta` records among the `stream_event` + entries. Zero means the proxy collapsed the streamed tool input + into a single complete block instead of forwarding the incremental + deltas the upstream emitted.""" + inner_events = ( + event.get("event") for event in events if event.get("type") == "stream_event" + ) + return sum( + 1 + for inner in inner_events + if isinstance(inner, Mapping) + and inner.get("type") == "content_block_delta" + and isinstance(inner.get("delta"), Mapping) + and inner["delta"].get("type") == "input_json_delta" + ) + + +def test_tool_use_streaming_bedrock_converse(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _count_input_json_deltas(outcome.events) == 0: + error = ( + f"[{model}] no input_json_delta stream events observed; proxy " + f"likely buffered the tool input into a complete block or " + f"stripped fine-grained tool streaming" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py new file mode 100644 index 00000000000..af4689b2847 --- /dev/null +++ b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py @@ -0,0 +1,152 @@ +"""tool_use_streaming x Bedrock (Invoke). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Claude requests to +AWS Bedrock via the legacy `InvokeModel` API path, ask Claude to invoke +a built-in tool (`Bash`), and assert that the upstream (a) emitted a +`tool_use` content block and (b) actually streamed events incrementally. + +Bedrock InvokeModel surfaces tool-streaming via `InvokeModelWithResponseStream`; +this cell catches gateway regressions where the proxy buffers the +response or fails to translate the streaming envelope to Anthropic +`message_*` event shape. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", + "--include-partial-messages", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: + """Count `input_json_delta` records among the `stream_event` + entries. Zero means the proxy collapsed the streamed tool input + into a single complete block instead of forwarding the incremental + deltas the upstream emitted.""" + inner_events = ( + event.get("event") for event in events if event.get("type") == "stream_event" + ) + return sum( + 1 + for inner in inner_events + if isinstance(inner, Mapping) + and inner.get("type") == "content_block_delta" + and isinstance(inner.get("delta"), Mapping) + and inner["delta"].get("type") == "input_json_delta" + ) + + +def test_tool_use_streaming_bedrock_invoke(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _count_input_json_deltas(outcome.events) == 0: + error = ( + f"[{model}] no input_json_delta stream events observed; proxy " + f"likely buffered the tool input into a complete block or " + f"stripped fine-grained tool streaming" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py new file mode 100644 index 00000000000..19ef9a4e90e --- /dev/null +++ b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py @@ -0,0 +1,151 @@ +"""tool_use_streaming x Vertex AI. + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Claude requests to +GCP Vertex AI, ask Claude to invoke a built-in tool (`Bash`), and +assert that the upstream (a) emitted a `tool_use` content block and +(b) actually streamed events incrementally. + +Vertex AI exposes Anthropic models via `:streamRawPredict`; this cell +catches gateway regressions where the proxy buffers the response or +strips the streaming beta header on the way to Vertex. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", + "--include-partial-messages", +] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: + """Count `input_json_delta` records among the `stream_event` + entries. Zero means the proxy collapsed the streamed tool input + into a single complete block instead of forwarding the incremental + deltas the upstream emitted.""" + inner_events = ( + event.get("event") for event in events if event.get("type") == "stream_event" + ) + return sum( + 1 + for inner in inner_events + if isinstance(inner, Mapping) + and inner.get("type") == "content_block_delta" + and isinstance(inner.get("delta"), Mapping) + and inner["delta"].get("type") == "input_json_delta" + ) + + +def test_tool_use_streaming_vertex_ai(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _count_input_json_deltas(outcome.events) == 0: + error = ( + f"[{model}] no input_json_delta stream events observed; proxy " + f"likely buffered the tool input into a complete block or " + f"stripped fine-grained tool streaming" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/vision/__init__.py b/tests/e2e/claude_code/vision/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/vision/test_anthropic.py b/tests/e2e/claude_code/vision/test_anthropic.py new file mode 100644 index 00000000000..650940248ea --- /dev/null +++ b/tests/e2e/claude_code/vision/test_anthropic.py @@ -0,0 +1,142 @@ +"""vision x Anthropic. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Anthropic, attach a small image as an inline base64 `image` content +block via the CLI's `--input-format stream-json` mode, and assert that +the upstream produces a non-empty reply. This proves the proxy +preserves Claude Code's multimodal content blocks end-to-end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/vision/test_anthropic.py + ^^^^^^ ^^^^^^^^^ + feature_id provider + +Why stream-json input rather than `--image `: the claude CLI +dropped `--image` in 2.x. Image attachments are now driven via either +the Files API (server-uploaded blobs referenced by file_id) or by +sending an Anthropic-shaped user message through stdin. We use the +latter because it requires no upstream pre-upload — the test stays +hermetic and the wire shape (an `image` content block) is exactly what +the proxy must preserve. +""" + +from __future__ import annotations + +import json +import os + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +# Minimal 1x1 red PNG, base64-encoded. We embed it directly as the +# `image` content block's source — no temp file or Files API upload +# needed, the test stays hermetic. +RED_PIXEL_PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8" + "z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) + +VISION_PROMPT = ( + "What single color do you see in the attached image? Answer in one word." +) + + +def _build_stdin_input() -> str: + """Build the newline-delimited JSON payload for `--input-format stream-json`. + + The CLI consumes a stream of `user` events whose `message.content` is + a list of Anthropic content blocks. A single user event with one + text block + one image block is enough to exercise the multimodal + code path. + """ + user_event = { + "type": "user", + "message": { + "role": "user", + "content": [ + {"type": "text", "text": VISION_PROMPT}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": RED_PIXEL_PNG_B64, + }, + }, + ], + }, + } + return json.dumps(user_event) + "\n" + + +def test_vision_anthropic(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy with an image + attached via stream-json input and assert a non-empty reply.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + # When using --input-format stream-json the CLI rejects a + # positional prompt; the prompt + image come in via stdin. + prompt=None, + base_url=base_url, + api_key=api_key, + extra_args=["--input-format", "stream-json"], + stdin_input=_build_stdin_input(), + ) + + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text on a vision prompt" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/vision/test_azure.py b/tests/e2e/claude_code/vision/test_azure.py new file mode 100644 index 00000000000..3b03c0f2b35 --- /dev/null +++ b/tests/e2e/claude_code/vision/test_azure.py @@ -0,0 +1,142 @@ +"""vision x Azure. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Azure, attach a small image as an inline base64 `image` content +block via the CLI's `--input-format stream-json` mode, and assert that +the upstream produces a non-empty reply. This proves the proxy +preserves Claude Code's multimodal content blocks end-to-end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/vision/test_azure.py + ^^^^^^ ^^^^^^^^^ + feature_id provider + +Why stream-json input rather than `--image `: the claude CLI +dropped `--image` in 2.x. Image attachments are now driven via either +the Files API (server-uploaded blobs referenced by file_id) or by +sending an Anthropic-shaped user message through stdin. We use the +latter because it requires no upstream pre-upload — the test stays +hermetic and the wire shape (an `image` content block) is exactly what +the proxy must preserve. +""" + +from __future__ import annotations + +import json +import os + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + +# Minimal 1x1 red PNG, base64-encoded. We embed it directly as the +# `image` content block's source — no temp file or Files API upload +# needed, the test stays hermetic. +RED_PIXEL_PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8" + "z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) + +VISION_PROMPT = ( + "What single color do you see in the attached image? Answer in one word." +) + + +def _build_stdin_input() -> str: + """Build the newline-delimited JSON payload for `--input-format stream-json`. + + The CLI consumes a stream of `user` events whose `message.content` is + a list of Anthropic content blocks. A single user event with one + text block + one image block is enough to exercise the multimodal + code path. + """ + user_event = { + "type": "user", + "message": { + "role": "user", + "content": [ + {"type": "text", "text": VISION_PROMPT}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": RED_PIXEL_PNG_B64, + }, + }, + ], + }, + } + return json.dumps(user_event) + "\n" + + +def test_vision_azure(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy with an image + attached via stream-json input and assert a non-empty reply.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + # When using --input-format stream-json the CLI rejects a + # positional prompt; the prompt + image come in via stdin. + prompt=None, + base_url=base_url, + api_key=api_key, + extra_args=["--input-format", "stream-json"], + stdin_input=_build_stdin_input(), + ) + + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text on a vision prompt" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/vision/test_bedrock_converse.py b/tests/e2e/claude_code/vision/test_bedrock_converse.py new file mode 100644 index 00000000000..4201f9e64fc --- /dev/null +++ b/tests/e2e/claude_code/vision/test_bedrock_converse.py @@ -0,0 +1,142 @@ +"""vision x Bedrock Converse. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Bedrock Converse, attach a small image as an inline base64 `image` content +block via the CLI's `--input-format stream-json` mode, and assert that +the upstream produces a non-empty reply. This proves the proxy +preserves Claude Code's multimodal content blocks end-to-end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/vision/test_bedrock_converse.py + ^^^^^^ ^^^^^^^^^ + feature_id provider + +Why stream-json input rather than `--image `: the claude CLI +dropped `--image` in 2.x. Image attachments are now driven via either +the Files API (server-uploaded blobs referenced by file_id) or by +sending an Anthropic-shaped user message through stdin. We use the +latter because it requires no upstream pre-upload — the test stays +hermetic and the wire shape (an `image` content block) is exactly what +the proxy must preserve. +""" + +from __future__ import annotations + +import json +import os + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + +# Minimal 1x1 red PNG, base64-encoded. We embed it directly as the +# `image` content block's source — no temp file or Files API upload +# needed, the test stays hermetic. +RED_PIXEL_PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8" + "z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) + +VISION_PROMPT = ( + "What single color do you see in the attached image? Answer in one word." +) + + +def _build_stdin_input() -> str: + """Build the newline-delimited JSON payload for `--input-format stream-json`. + + The CLI consumes a stream of `user` events whose `message.content` is + a list of Anthropic content blocks. A single user event with one + text block + one image block is enough to exercise the multimodal + code path. + """ + user_event = { + "type": "user", + "message": { + "role": "user", + "content": [ + {"type": "text", "text": VISION_PROMPT}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": RED_PIXEL_PNG_B64, + }, + }, + ], + }, + } + return json.dumps(user_event) + "\n" + + +def test_vision_bedrock_converse(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy with an image + attached via stream-json input and assert a non-empty reply.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + # When using --input-format stream-json the CLI rejects a + # positional prompt; the prompt + image come in via stdin. + prompt=None, + base_url=base_url, + api_key=api_key, + extra_args=["--input-format", "stream-json"], + stdin_input=_build_stdin_input(), + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text on a vision prompt" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/vision/test_bedrock_invoke.py b/tests/e2e/claude_code/vision/test_bedrock_invoke.py new file mode 100644 index 00000000000..d2e641f1462 --- /dev/null +++ b/tests/e2e/claude_code/vision/test_bedrock_invoke.py @@ -0,0 +1,142 @@ +"""vision x Bedrock Invoke. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Bedrock Invoke, attach a small image as an inline base64 `image` content +block via the CLI's `--input-format stream-json` mode, and assert that +the upstream produces a non-empty reply. This proves the proxy +preserves Claude Code's multimodal content blocks end-to-end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/vision/test_bedrock_invoke.py + ^^^^^^ ^^^^^^^^^ + feature_id provider + +Why stream-json input rather than `--image `: the claude CLI +dropped `--image` in 2.x. Image attachments are now driven via either +the Files API (server-uploaded blobs referenced by file_id) or by +sending an Anthropic-shaped user message through stdin. We use the +latter because it requires no upstream pre-upload — the test stays +hermetic and the wire shape (an `image` content block) is exactly what +the proxy must preserve. +""" + +from __future__ import annotations + +import json +import os + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + +# Minimal 1x1 red PNG, base64-encoded. We embed it directly as the +# `image` content block's source — no temp file or Files API upload +# needed, the test stays hermetic. +RED_PIXEL_PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8" + "z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) + +VISION_PROMPT = ( + "What single color do you see in the attached image? Answer in one word." +) + + +def _build_stdin_input() -> str: + """Build the newline-delimited JSON payload for `--input-format stream-json`. + + The CLI consumes a stream of `user` events whose `message.content` is + a list of Anthropic content blocks. A single user event with one + text block + one image block is enough to exercise the multimodal + code path. + """ + user_event = { + "type": "user", + "message": { + "role": "user", + "content": [ + {"type": "text", "text": VISION_PROMPT}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": RED_PIXEL_PNG_B64, + }, + }, + ], + }, + } + return json.dumps(user_event) + "\n" + + +def test_vision_bedrock_invoke(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy with an image + attached via stream-json input and assert a non-empty reply.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + # When using --input-format stream-json the CLI rejects a + # positional prompt; the prompt + image come in via stdin. + prompt=None, + base_url=base_url, + api_key=api_key, + extra_args=["--input-format", "stream-json"], + stdin_input=_build_stdin_input(), + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text on a vision prompt" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/vision/test_vertex_ai.py b/tests/e2e/claude_code/vision/test_vertex_ai.py new file mode 100644 index 00000000000..a39ef1a34b7 --- /dev/null +++ b/tests/e2e/claude_code/vision/test_vertex_ai.py @@ -0,0 +1,142 @@ +"""vision x Vertex AI. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Vertex AI, attach a small image as an inline base64 `image` content +block via the CLI's `--input-format stream-json` mode, and assert that +the upstream produces a non-empty reply. This proves the proxy +preserves Claude Code's multimodal content blocks end-to-end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/vision/test_vertex_ai.py + ^^^^^^ ^^^^^^^^^ + feature_id provider + +Why stream-json input rather than `--image `: the claude CLI +dropped `--image` in 2.x. Image attachments are now driven via either +the Files API (server-uploaded blobs referenced by file_id) or by +sending an Anthropic-shaped user message through stdin. We use the +latter because it requires no upstream pre-upload — the test stays +hermetic and the wire shape (an `image` content block) is exactly what +the proxy must preserve. +""" + +from __future__ import annotations + +import json +import os + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + +# Minimal 1x1 red PNG, base64-encoded. We embed it directly as the +# `image` content block's source — no temp file or Files API upload +# needed, the test stays hermetic. +RED_PIXEL_PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8" + "z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) + +VISION_PROMPT = ( + "What single color do you see in the attached image? Answer in one word." +) + + +def _build_stdin_input() -> str: + """Build the newline-delimited JSON payload for `--input-format stream-json`. + + The CLI consumes a stream of `user` events whose `message.content` is + a list of Anthropic content blocks. A single user event with one + text block + one image block is enough to exercise the multimodal + code path. + """ + user_event = { + "type": "user", + "message": { + "role": "user", + "content": [ + {"type": "text", "text": VISION_PROMPT}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": RED_PIXEL_PNG_B64, + }, + }, + ], + }, + } + return json.dumps(user_event) + "\n" + + +def test_vision_vertex_ai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy with an image + attached via stream-json input and assert a non-empty reply.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + # When using --input-format stream-json the CLI rejects a + # positional prompt; the prompt + image come in via stdin. + prompt=None, + base_url=base_url, + api_key=api_key, + extra_args=["--input-format", "stream-json"], + stdin_input=_build_stdin_input(), + ) + + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text on a vision prompt" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/web_search/__init__.py b/tests/e2e/claude_code/web_search/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/web_search/test_anthropic.py b/tests/e2e/claude_code/web_search/test_anthropic.py new file mode 100644 index 00000000000..b8fa806f923 --- /dev/null +++ b/tests/e2e/claude_code/web_search/test_anthropic.py @@ -0,0 +1,146 @@ +"""web_search x Anthropic. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Anthropic, allow the built-in `WebSearch` tool, ask a question that +requires fresh web data, and assert that the upstream emitted a +`tool_use` block calling `WebSearch` — proving the proxy preserves +Claude Code's tool definitions and the upstream's tool-use response +end-to-end. + +Note: Claude Code's `WebSearch` is a *client-side* tool (the CLI +executes the search itself and feeds the result back as a `tool_result` +block), so the wire shape is `tool_use` with `name="WebSearch"` rather +than the Anthropic-managed `server_tool_use` / `web_search_tool_result` +blocks (which only appear when the request includes the +`web_search_20250305` server tool definition — something the CLI does +not currently inject). A regression where the proxy strips the +`WebSearch` tool from the request or drops the `tool_use` block from +the response will break this assertion. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/web_search/test_anthropic.py + ^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +# A prompt the model cannot answer from training data alone — it forces +# the model to actually hit the web_search server tool rather than +# replying from memory. We pick "this week" as the freshness anchor +# because it's stable across long-running test schedules without +# pinning to a specific date that would go stale. +WEB_SEARCH_PROMPT = ( + "Use web search to find a news headline published this week about " + "Anthropic. Reply with one sentence summarizing what you found." +) +# Allow only WebSearch so the model has no fallback path: if the proxy +# strips the server tool, the run will fail loudly rather than silently +# answering from training data via a different tool. +WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"] + +# The CLI tool name surfaced as `tool_use.name` when WebSearch fires. +WEB_SEARCH_TOOL_NAME = "WebSearch" + + +def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: + """Walk the stream-json events and return True if any assistant + message included a `tool_use` block calling `WebSearch`.""" + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + if ( + block.get("type") == "tool_use" + and block.get("name") == WEB_SEARCH_TOOL_NAME + ): + return True + return False + + +def test_web_search_anthropic(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert the + upstream emitted a `tool_use` block calling `WebSearch`, proving + the proxy preserved both the request-side tool definition and the + response-side tool_use block.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt=WEB_SEARCH_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=WEB_SEARCH_ARGS, + ) + + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_web_search_tool_use(outcome.events): + error = ( + f"[{model}] no `tool_use` block with name=WebSearch observed; " + "the proxy may have stripped the WebSearch tool definition from " + "the request or the tool_use block from the response" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/web_search/test_azure.py b/tests/e2e/claude_code/web_search/test_azure.py new file mode 100644 index 00000000000..e70dc848dcf --- /dev/null +++ b/tests/e2e/claude_code/web_search/test_azure.py @@ -0,0 +1,146 @@ +"""web_search x Azure. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Azure, allow the built-in `WebSearch` tool, ask a question that +requires fresh web data, and assert that the upstream emitted a +`tool_use` block calling `WebSearch` — proving the proxy preserves +Claude Code's tool definitions and the upstream's tool-use response +end-to-end. + +Note: Claude Code's `WebSearch` is a *client-side* tool (the CLI +executes the search itself and feeds the result back as a `tool_result` +block), so the wire shape is `tool_use` with `name="WebSearch"` rather +than the Anthropic-managed `server_tool_use` / `web_search_tool_result` +blocks (which only appear when the request includes the +`web_search_20250305` server tool definition — something the CLI does +not currently inject). A regression where the proxy strips the +`WebSearch` tool from the request or drops the `tool_use` block from +the response will break this assertion. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/web_search/test_azure.py + ^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + +# A prompt the model cannot answer from training data alone — it forces +# the model to actually hit the web_search server tool rather than +# replying from memory. We pick "this week" as the freshness anchor +# because it's stable across long-running test schedules without +# pinning to a specific date that would go stale. +WEB_SEARCH_PROMPT = ( + "Use web search to find a news headline published this week about " + "Anthropic. Reply with one sentence summarizing what you found." +) +# Allow only WebSearch so the model has no fallback path: if the proxy +# strips the server tool, the run will fail loudly rather than silently +# answering from training data via a different tool. +WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"] + +# The CLI tool name surfaced as `tool_use.name` when WebSearch fires. +WEB_SEARCH_TOOL_NAME = "WebSearch" + + +def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: + """Walk the stream-json events and return True if any assistant + message included a `tool_use` block calling `WebSearch`.""" + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + if ( + block.get("type") == "tool_use" + and block.get("name") == WEB_SEARCH_TOOL_NAME + ): + return True + return False + + +def test_web_search_azure(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert the + upstream emitted a `tool_use` block calling `WebSearch`, proving + the proxy preserved both the request-side tool definition and the + response-side tool_use block.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt=WEB_SEARCH_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=WEB_SEARCH_ARGS, + ) + + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_web_search_tool_use(outcome.events): + error = ( + f"[{model}] no `tool_use` block with name=WebSearch observed; " + "the proxy may have stripped the WebSearch tool definition from " + "the request or the tool_use block from the response" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/web_search/test_bedrock_converse.py b/tests/e2e/claude_code/web_search/test_bedrock_converse.py new file mode 100644 index 00000000000..cbeea03df40 --- /dev/null +++ b/tests/e2e/claude_code/web_search/test_bedrock_converse.py @@ -0,0 +1,146 @@ +"""web_search x Bedrock Converse. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Bedrock Converse, allow the built-in `WebSearch` tool, ask a question that +requires fresh web data, and assert that the upstream emitted a +`tool_use` block calling `WebSearch` — proving the proxy preserves +Claude Code's tool definitions and the upstream's tool-use response +end-to-end. + +Note: Claude Code's `WebSearch` is a *client-side* tool (the CLI +executes the search itself and feeds the result back as a `tool_result` +block), so the wire shape is `tool_use` with `name="WebSearch"` rather +than the Anthropic-managed `server_tool_use` / `web_search_tool_result` +blocks (which only appear when the request includes the +`web_search_20250305` server tool definition — something the CLI does +not currently inject). A regression where the proxy strips the +`WebSearch` tool from the request or drops the `tool_use` block from +the response will break this assertion. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/web_search/test_bedrock_converse.py + ^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + +# A prompt the model cannot answer from training data alone — it forces +# the model to actually hit the web_search server tool rather than +# replying from memory. We pick "this week" as the freshness anchor +# because it's stable across long-running test schedules without +# pinning to a specific date that would go stale. +WEB_SEARCH_PROMPT = ( + "Use web search to find a news headline published this week about " + "Anthropic. Reply with one sentence summarizing what you found." +) +# Allow only WebSearch so the model has no fallback path: if the proxy +# strips the server tool, the run will fail loudly rather than silently +# answering from training data via a different tool. +WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"] + +# The CLI tool name surfaced as `tool_use.name` when WebSearch fires. +WEB_SEARCH_TOOL_NAME = "WebSearch" + + +def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: + """Walk the stream-json events and return True if any assistant + message included a `tool_use` block calling `WebSearch`.""" + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + if ( + block.get("type") == "tool_use" + and block.get("name") == WEB_SEARCH_TOOL_NAME + ): + return True + return False + + +def test_web_search_bedrock_converse(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert the + upstream emitted a `tool_use` block calling `WebSearch`, proving + the proxy preserved both the request-side tool definition and the + response-side tool_use block.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt=WEB_SEARCH_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=WEB_SEARCH_ARGS, + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_web_search_tool_use(outcome.events): + error = ( + f"[{model}] no `tool_use` block with name=WebSearch observed; " + "the proxy may have stripped the WebSearch tool definition from " + "the request or the tool_use block from the response" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/web_search/test_bedrock_invoke.py b/tests/e2e/claude_code/web_search/test_bedrock_invoke.py new file mode 100644 index 00000000000..86068e1e22b --- /dev/null +++ b/tests/e2e/claude_code/web_search/test_bedrock_invoke.py @@ -0,0 +1,146 @@ +"""web_search x Bedrock Invoke. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Bedrock Invoke, allow the built-in `WebSearch` tool, ask a question that +requires fresh web data, and assert that the upstream emitted a +`tool_use` block calling `WebSearch` — proving the proxy preserves +Claude Code's tool definitions and the upstream's tool-use response +end-to-end. + +Note: Claude Code's `WebSearch` is a *client-side* tool (the CLI +executes the search itself and feeds the result back as a `tool_result` +block), so the wire shape is `tool_use` with `name="WebSearch"` rather +than the Anthropic-managed `server_tool_use` / `web_search_tool_result` +blocks (which only appear when the request includes the +`web_search_20250305` server tool definition — something the CLI does +not currently inject). A regression where the proxy strips the +`WebSearch` tool from the request or drops the `tool_use` block from +the response will break this assertion. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/web_search/test_bedrock_invoke.py + ^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + +# A prompt the model cannot answer from training data alone — it forces +# the model to actually hit the web_search server tool rather than +# replying from memory. We pick "this week" as the freshness anchor +# because it's stable across long-running test schedules without +# pinning to a specific date that would go stale. +WEB_SEARCH_PROMPT = ( + "Use web search to find a news headline published this week about " + "Anthropic. Reply with one sentence summarizing what you found." +) +# Allow only WebSearch so the model has no fallback path: if the proxy +# strips the server tool, the run will fail loudly rather than silently +# answering from training data via a different tool. +WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"] + +# The CLI tool name surfaced as `tool_use.name` when WebSearch fires. +WEB_SEARCH_TOOL_NAME = "WebSearch" + + +def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: + """Walk the stream-json events and return True if any assistant + message included a `tool_use` block calling `WebSearch`.""" + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + if ( + block.get("type") == "tool_use" + and block.get("name") == WEB_SEARCH_TOOL_NAME + ): + return True + return False + + +def test_web_search_bedrock_invoke(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert the + upstream emitted a `tool_use` block calling `WebSearch`, proving + the proxy preserved both the request-side tool definition and the + response-side tool_use block.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt=WEB_SEARCH_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=WEB_SEARCH_ARGS, + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_web_search_tool_use(outcome.events): + error = ( + f"[{model}] no `tool_use` block with name=WebSearch observed; " + "the proxy may have stripped the WebSearch tool definition from " + "the request or the tool_use block from the response" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/e2e/claude_code/web_search/test_vertex_ai.py b/tests/e2e/claude_code/web_search/test_vertex_ai.py new file mode 100644 index 00000000000..a33515771f3 --- /dev/null +++ b/tests/e2e/claude_code/web_search/test_vertex_ai.py @@ -0,0 +1,146 @@ +"""web_search x Vertex AI. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Vertex AI, allow the built-in `WebSearch` tool, ask a question that +requires fresh web data, and assert that the upstream emitted a +`tool_use` block calling `WebSearch` — proving the proxy preserves +Claude Code's tool definitions and the upstream's tool-use response +end-to-end. + +Note: Claude Code's `WebSearch` is a *client-side* tool (the CLI +executes the search itself and feeds the result back as a `tool_result` +block), so the wire shape is `tool_use` with `name="WebSearch"` rather +than the Anthropic-managed `server_tool_use` / `web_search_tool_result` +blocks (which only appear when the request includes the +`web_search_20250305` server tool definition — something the CLI does +not currently inject). A regression where the proxy strips the +`WebSearch` tool from the request or drops the `tool_use` block from +the response will break this assertion. + +The (feature, provider) for this cell is inferred from the file path by +`tests/e2e/claude_code/conftest.py`: + + tests/e2e/claude_code/web_search/test_vertex_ai.py + ^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + +# A prompt the model cannot answer from training data alone — it forces +# the model to actually hit the web_search server tool rather than +# replying from memory. We pick "this week" as the freshness anchor +# because it's stable across long-running test schedules without +# pinning to a specific date that would go stale. +WEB_SEARCH_PROMPT = ( + "Use web search to find a news headline published this week about " + "Anthropic. Reply with one sentence summarizing what you found." +) +# Allow only WebSearch so the model has no fallback path: if the proxy +# strips the server tool, the run will fail loudly rather than silently +# answering from training data via a different tool. +WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"] + +# The CLI tool name surfaced as `tool_use.name` when WebSearch fires. +WEB_SEARCH_TOOL_NAME = "WebSearch" + + +def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: + """Walk the stream-json events and return True if any assistant + message included a `tool_use` block calling `WebSearch`.""" + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + if ( + block.get("type") == "tool_use" + and block.get("name") == WEB_SEARCH_TOOL_NAME + ): + return True + return False + + +def test_web_search_vertex_ai(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert the + upstream emitted a `tool_use` block calling `WebSearch`, proving + the proxy preserved both the request-side tool definition and the + response-side tool_use block.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt=WEB_SEARCH_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=WEB_SEARCH_ARGS, + ) + + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_web_search_tool_use(outcome.events): + error = ( + f"[{model}] no `tool_use` block with name=WebSearch observed; " + "the proxy may have stripped the WebSearch tool definition from " + "the request or the tool_use block from the response" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False)