diff --git a/tests/claude_code/_basic_messaging.py b/tests/claude_code/_basic_messaging.py index 80e9827ba8e..4e23d5f1ed2 100644 --- a/tests/claude_code/_basic_messaging.py +++ b/tests/claude_code/_basic_messaging.py @@ -5,17 +5,16 @@ Every basic_messaging cell follows the same skeleton: 1. Read the proxy base URL + API key from env, fail-early if missing. 2. Fan the three Claude tiers out via `run_claude_models_parallel`. 3. Inspect each model's outcome and report one `compat_result` row per - model — `ClaudeCLIError`, non-zero exit, missing stream events - (streaming variant only), and empty assistant text are all per-model - fails; everything else is a per-model pass. + model — `ClaudeCLIError`, non-zero exit, and empty assistant text + are all per-model fails; everything else is a per-model pass. 4. Surface a joined failure message via `pytest.fail(...)` so the pytest run also goes red. The conftest infers `(feature_id, provider)` purely from the test file path, so each per-provider file just declares its model list and calls `run_basic_messaging_cell(...)`. This keeps all cell logic in one place -— a future tweak to the env-missing guard, the failure-loop shape, or -the stream-events check now propagates to every cell automatically. +— a future tweak to the env-missing guard or the failure-loop shape +now propagates to every cell automatically. The leading underscore in the filename is what keeps pytest from collecting this module as a test file. @@ -43,14 +42,17 @@ def run_basic_messaging_cell( compat_result, models: Sequence[str], prompt: str, - require_stream_events: bool = False, ) -> None: """Run the shared `basic_messaging_*` × cell body. - `require_stream_events=True` adds the streaming-variant assertion - that at least one stream-json event is observed for each model — - the regression check that catches a proxy buffering the full - response before flushing. + The streaming and non-streaming variants share this body because + the CLI driver consumes stdout via `subprocess.run(capture_output=True)` + after the process exits — we can only observe that events arrived, + not *when* they arrived. A wire-level "did the proxy buffer the + full response before flushing?" check therefore can't live here; + it belongs in a driver that streams stdout incrementally. Until + that exists, the streaming cells exercise the same shape and + check the same per-model outcomes as the non-streaming cells. """ base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) @@ -91,12 +93,6 @@ def run_basic_messaging_cell( failures.append(error) continue - if require_stream_events and not outcome.events: - error = f"[{model}] no stream-json events emitted; streaming wire silent" - compat_result.add({"status": "fail", "error": error}) - failures.append(error) - continue - if not outcome.text.strip(): error = f"[{model}] claude returned empty assistant text" compat_result.add({"status": "fail", "error": error}) diff --git a/tests/claude_code/basic_messaging_streaming/test_anthropic.py b/tests/claude_code/basic_messaging_streaming/test_anthropic.py index 378af3bceb2..7d7a5bb32e2 100644 --- a/tests/claude_code/basic_messaging_streaming/test_anthropic.py +++ b/tests/claude_code/basic_messaging_streaming/test_anthropic.py @@ -6,10 +6,16 @@ report the outcome via `compat_result`. The CLI is run with `--print --output-format stream-json`, which streams incremental events as the upstream produces tokens. The cell goes green -only when every Claude tier returns a non-empty reply over a streamed -wire (i.e. at least one stream-json event is observed). This catches -regressions where the proxy buffers the full response before flushing, -silently degrading the streaming experience customers rely on. +only when every Claude tier returns a non-empty reply. + +Note: a true "did the proxy buffer the full response before flushing?" +check would require observing event arrival times on the wire, which +the `cli_driver` cannot do today — it consumes stdout via +`subprocess.run(capture_output=True)` after the process exits, so a +buffered-then-flushed response is indistinguishable from a truly +streamed one. That regression check belongs in a streaming-aware +driver; until then this cell verifies the same shape as the +non-streaming variant. The (feature, provider) for this cell is inferred from the file path by `tests/claude_code/conftest.py`: @@ -21,9 +27,7 @@ The (feature, provider) for this cell is inferred from the file path by The shared `run_basic_messaging_cell` helper fans the three Claude tiers out in parallel inside this single test, with one `compat_result.add(...)` entry per model so the matrix builder still -sees three rows for this (feature, provider). The `require_stream_events` -flag adds the streaming-only assertion that at least one stream-json -event was observed per model. +sees three rows for this (feature, provider). """ from __future__ import annotations @@ -39,11 +43,10 @@ ANTHROPIC_MODELS = [ def test_basic_messaging_streaming_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a - non-empty streamed reply (at least one stream-json event observed). + non-empty streamed reply (one row per Claude tier). """ run_basic_messaging_cell( compat_result=compat_result, models=ANTHROPIC_MODELS, prompt="Count from 1 to 5, one number per line.", - require_stream_events=True, ) diff --git a/tests/claude_code/basic_messaging_streaming/test_azure.py b/tests/claude_code/basic_messaging_streaming/test_azure.py index bea1a3b5a2d..f7d344615c8 100644 --- a/tests/claude_code/basic_messaging_streaming/test_azure.py +++ b/tests/claude_code/basic_messaging_streaming/test_azure.py @@ -30,11 +30,10 @@ AZURE_MODELS = [ def test_basic_messaging_streaming_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a - non-empty streamed reply (at least one stream-json event observed). + non-empty streamed reply (one row per Claude tier). """ run_basic_messaging_cell( compat_result=compat_result, models=AZURE_MODELS, prompt="Count from 1 to 5, one number per line.", - require_stream_events=True, ) diff --git a/tests/claude_code/basic_messaging_streaming/test_bedrock_converse.py b/tests/claude_code/basic_messaging_streaming/test_bedrock_converse.py index 5e655a617f9..3a8a81bfc5b 100644 --- a/tests/claude_code/basic_messaging_streaming/test_bedrock_converse.py +++ b/tests/claude_code/basic_messaging_streaming/test_bedrock_converse.py @@ -26,11 +26,10 @@ BEDROCK_CONVERSE_MODELS = [ def test_basic_messaging_streaming_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a - non-empty streamed reply (at least one stream-json event observed). + non-empty streamed reply (one row per Claude tier). """ run_basic_messaging_cell( compat_result=compat_result, models=BEDROCK_CONVERSE_MODELS, prompt="Count from 1 to 5, one number per line.", - require_stream_events=True, ) diff --git a/tests/claude_code/basic_messaging_streaming/test_bedrock_invoke.py b/tests/claude_code/basic_messaging_streaming/test_bedrock_invoke.py index aeb43095df6..a24bb7f1ef4 100644 --- a/tests/claude_code/basic_messaging_streaming/test_bedrock_invoke.py +++ b/tests/claude_code/basic_messaging_streaming/test_bedrock_invoke.py @@ -26,11 +26,10 @@ BEDROCK_INVOKE_MODELS = [ def test_basic_messaging_streaming_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a - non-empty streamed reply (at least one stream-json event observed). + non-empty streamed reply (one row per Claude tier). """ run_basic_messaging_cell( compat_result=compat_result, models=BEDROCK_INVOKE_MODELS, prompt="Count from 1 to 5, one number per line.", - require_stream_events=True, ) diff --git a/tests/claude_code/basic_messaging_streaming/test_vertex_ai.py b/tests/claude_code/basic_messaging_streaming/test_vertex_ai.py index 6f00c7d330f..a369c780ebc 100644 --- a/tests/claude_code/basic_messaging_streaming/test_vertex_ai.py +++ b/tests/claude_code/basic_messaging_streaming/test_vertex_ai.py @@ -26,11 +26,10 @@ VERTEX_AI_MODELS = [ def test_basic_messaging_streaming_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a - non-empty streamed reply (at least one stream-json event observed). + non-empty streamed reply (one row per Claude tier). """ run_basic_messaging_cell( compat_result=compat_result, models=VERTEX_AI_MODELS, prompt="Count from 1 to 5, one number per line.", - require_stream_events=True, ) diff --git a/tests/claude_code/conftest.py b/tests/claude_code/conftest.py index 6f3118df664..89bf5bfc529 100644 --- a/tests/claude_code/conftest.py +++ b/tests/claude_code/conftest.py @@ -241,6 +241,16 @@ def pytest_runtest_makereport(item, call): elif report.when != "call": return + # A skipped test (e.g. `pytest.skip(...)` called inside the body or + # by a `pytest.mark.skipif` evaluated at call time) is neither a + # pass nor a fail — it just didn't run. Recording it as anything + # here would produce a spurious row (the not-failed/empty-collected + # branch below would mark it as a fail with "test passed without + # reporting via compat_result"), so bail out and let the cell stay + # "not_tested" in the published matrix. + if report.skipped: + return + inferred = _infer_feature_and_provider(Path(str(item.path))) if inferred is None: return