From 348ecf655668091a53259421154bcdeeb30bb320 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 22 Sep 2026 11:03:30 -0700 Subject: [PATCH] test(e2e): register client-apps deployments per run and make the turns cache-proof The suite now creates its own Anthropic and OpenAI deployments through /model/new under run-unique names and deletes them at session end, so it no longer depends on which aliases the proxy config serves. The names start with the real model id because Claude Code 2.1.278 picks its request shape from the model name and sends an unrecognized one a system-role probe that Anthropic rejects with a 400 before the CLI falls back, which the proxy bills as a failure row. The echoed word is unique per run: the CI stack caches responses, and an identical prompt across the three CI passes was served a $0 cache-hit row. Tests move into the classes the e2e harness requires, the spend helper asserts the row count it polls for, and the AI SDK cell accepts a single text delta since the provider chunks the final sentence as it likes. --- tests/e2e/client_apps/conftest.py | 40 ++++++++- tests/e2e/client_apps/spend_rows.py | 1 + .../client_apps/test_claude_code_cli_e2e.py | 89 ++++++++++--------- tests/e2e/client_apps/test_codex_cli_e2e.py | 49 +++++----- .../e2e/client_apps/test_vercel_ai_sdk_e2e.py | 63 ++++++------- 5 files changed, 148 insertions(+), 94 deletions(-) diff --git a/tests/e2e/client_apps/conftest.py b/tests/e2e/client_apps/conftest.py index d35e8d7a597..b68945700bb 100644 --- a/tests/e2e/client_apps/conftest.py +++ b/tests/e2e/client_apps/conftest.py @@ -1,18 +1,56 @@ -"""`client_apps` suite's `client` fixture. +"""`client_apps` suite's `client` fixture and the two deployments it drives. Lifecycle (resources/scoped_key), proxy liveness gate, and the e2e/covers markers all live in the parent tests/e2e/conftest.py. ClientAppsClient holds the shared ProxyClient so the keys these tests mint tear down through it. + +The suite registers its own deployments through /model/new, one Anthropic and +one OpenAI, under names unique to the run, and deletes them at session end, so +it never depends on which aliases the proxy's config file serves. Each carries +the provider key from the test runner's env when set; otherwise an os.environ +reference the proxy resolves from its own env. They stay on the real provider +path (`provider_live`) because a coding client's prompts change every run, so a +recorded response could never stand in for them. Each deployment's name starts +with the real model id: Claude Code 2.1.278 keys its request shape on the model +name, and an unrecognized one gets a system-role message Anthropic rejects with a +400 before the CLI falls back, which the proxy would bill as a failure row. """ from __future__ import annotations +import os +from collections.abc import Iterator +from typing import Final + import pytest from client_apps_client import ClientAppsClient +from e2e_config import unique_marker +from models import LiteLLMParamsBody from proxy_client import ProxyClient @pytest.fixture(scope="session") def client(proxy: ProxyClient) -> ClientAppsClient: return ClientAppsClient(proxy=proxy) + + +@pytest.fixture(scope="session") +def anthropic_model(proxy: ProxyClient) -> Iterator[str]: + yield from _suite_model(proxy, "anthropic/claude-haiku-4-5", "ANTHROPIC_API_KEY") + + +@pytest.fixture(scope="session") +def openai_model(proxy: ProxyClient) -> Iterator[str]: + yield from _suite_model(proxy, "openai/gpt-5.5", "OPENAI_API_KEY") + + +def _suite_model(proxy: ProxyClient, provider_model: str, env_var: str) -> Iterator[str]: + model_name: Final = f"{provider_model.split('/', 1)[1]}-client-apps-{unique_marker()}" + model_id: Final = proxy.create_model( + model_name, + LiteLLMParamsBody(model=provider_model, api_key=os.environ.get(env_var) or f"os.environ/{env_var}"), + provider_live=True, + ) + yield model_name + proxy.delete_model(model_id) diff --git a/tests/e2e/client_apps/spend_rows.py b/tests/e2e/client_apps/spend_rows.py index 282b4250d57..05329d6e0b9 100644 --- a/tests/e2e/client_apps/spend_rows.py +++ b/tests/e2e/client_apps/spend_rows.py @@ -15,6 +15,7 @@ def recorded_request(row: SpendLogRow) -> RecordedRequest: def streamed_tool_turn_rows(proxy: ProxyClient, key: str) -> list[SpendLogRow]: rows = proxy.poll_logs_for_key(key, min_rows=2) + assert len(rows) >= 2, rows assert all(row.status == "success" for row in rows), rows assert all(row.spend is not None and row.spend > 0 for row in rows), rows assert all(recorded_request(row).stream for row in rows), rows diff --git a/tests/e2e/client_apps/test_claude_code_cli_e2e.py b/tests/e2e/client_apps/test_claude_code_cli_e2e.py index b3de2ea2a09..62c04d7c05a 100644 --- a/tests/e2e/client_apps/test_claude_code_cli_e2e.py +++ b/tests/e2e/client_apps/test_claude_code_cli_e2e.py @@ -2,12 +2,14 @@ turn against the proxy on a virtual key: it streams over `/v1/messages`, calls the Bash tool, reads the tool result, and answers. Client side the stream-json events carry the tool_use, the tool_result, the streamed deltas, and the final -result; proxy side the spend rows carry real spend and `stream: true`.""" +result; proxy side the spend rows carry real spend and `stream: true`. The +echoed word is unique per run: the CI stack caches responses, and an +identical prompt across runs would be served a $0 cache-hit row instead of a +real call.""" from __future__ import annotations import json -from typing import Final import pytest @@ -21,54 +23,57 @@ from client_apps_models import ( ToolResultBlock, ToolUseBlock, ) -from e2e_config import CHEAP_ANTHROPIC_MODEL +from e2e_config import unique_marker from spend_rows import streamed_tool_turn_rows pytestmark = pytest.mark.e2e -PROMPT: Final = "Use the Bash tool to run the command `echo pong` and report what it printed." - -@pytest.mark.covers("other.client_apps.claude_code_cli.tool_turn_streams") -def test_claude_code_cli_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str) -> None: - events = unwrap_run( - client.run_claude_code( - key=scoped_key, model=CHEAP_ANTHROPIC_MODEL, prompt=PROMPT, allowed_tool="Bash(echo pong)" +class TestClaudeCodeCli: + @pytest.mark.covers("other.client_apps.claude_code_cli.tool_turn_streams") + def test_streams_a_tool_turn(self, client: ClientAppsClient, scoped_key: str, anthropic_model: str) -> None: + word = f"pong-{unique_marker()}" + events = unwrap_run( + client.run_claude_code( + key=scoped_key, + model=anthropic_model, + prompt=f"Use the Bash tool to run the command `echo {word}` and report what it printed.", + allowed_tool=f"Bash(echo {word})", + ) ) - ) - tool_uses = [ - block - for event in events - if isinstance(event, ClaudeAssistantEvent) and isinstance(event.message.content, list) - for block in event.message.content - if isinstance(block, ToolUseBlock) - ] - assert [block.name for block in tool_uses] == ["Bash"], events + tool_uses = [ + block + for event in events + if isinstance(event, ClaudeAssistantEvent) and isinstance(event.message.content, list) + for block in event.message.content + if isinstance(block, ToolUseBlock) + ] + assert [block.name for block in tool_uses] == ["Bash"], events - tool_results = [ - block - for event in events - if isinstance(event, ClaudeUserEvent) and isinstance(event.message.content, list) - for block in event.message.content - if isinstance(block, ToolResultBlock) - ] - assert len(tool_results) == 1, events - assert not tool_results[0].is_error, tool_results[0] - assert "pong" in json.dumps(tool_results[0].content), tool_results[0] + tool_results = [ + block + for event in events + if isinstance(event, ClaudeUserEvent) and isinstance(event.message.content, list) + for block in event.message.content + if isinstance(block, ToolResultBlock) + ] + assert len(tool_results) == 1, events + assert not tool_results[0].is_error, tool_results[0] + assert word in json.dumps(tool_results[0].content), tool_results[0] - delta_types = [ - event.event.delta.type - for event in events - if isinstance(event, ClaudeStreamEvent) and isinstance(event.event, ContentBlockDelta) - ] - assert "input_json_delta" in delta_types, delta_types - assert "text_delta" in delta_types, delta_types + delta_types = [ + event.event.delta.type + for event in events + if isinstance(event, ClaudeStreamEvent) and isinstance(event.event, ContentBlockDelta) + ] + assert "input_json_delta" in delta_types, delta_types + assert "text_delta" in delta_types, delta_types - results = [event for event in events if isinstance(event, ClaudeResultEvent)] - assert len(results) == 1, events - assert results[0].subtype == "success", results[0] - assert not results[0].is_error, results[0] - assert results[0].result is not None and "pong" in results[0].result, results[0] + results = [event for event in events if isinstance(event, ClaudeResultEvent)] + assert len(results) == 1, events + assert results[0].subtype == "success", results[0] + assert not results[0].is_error, results[0] + assert results[0].result is not None and word in results[0].result, results[0] - streamed_tool_turn_rows(client.proxy, scoped_key) + streamed_tool_turn_rows(client.proxy, scoped_key) diff --git a/tests/e2e/client_apps/test_codex_cli_e2e.py b/tests/e2e/client_apps/test_codex_cli_e2e.py index 53ed2c4f686..5bcdadb0a43 100644 --- a/tests/e2e/client_apps/test_codex_cli_e2e.py +++ b/tests/e2e/client_apps/test_codex_cli_e2e.py @@ -4,6 +4,9 @@ model provider (`wire_api = "responses"`, so `/v1/responses`): it runs a shell command, reads its output, and answers. Client side the JSONL events carry the completed command execution and the agent message; proxy side the spend rows carry real spend and `stream: true` (Codex always streams the Responses API). +The echoed word is unique per run: the CI stack caches responses, and an +identical prompt across runs would be served a $0 cache-hit row instead of a +real call. The throwaway config turns Codex's sandbox off (`sandbox_mode = "danger-full-access"`): its Linux sandbox is bubblewrap, which cannot set up @@ -14,37 +17,41 @@ of value is reachable without the sandbox.""" from __future__ import annotations -from typing import Final - import pytest from client_apps_client import ClientAppsClient, unwrap_run from client_apps_models import CodexAgentMessage, CodexCommandExecution, CodexItemCompleted, CodexTurnCompleted -from e2e_config import CHEAP_OPENAI_MODEL +from e2e_config import unique_marker from spend_rows import streamed_tool_turn_rows pytestmark = pytest.mark.e2e -PROMPT: Final = "Run the shell command `echo pong` and reply with exactly what it printed." +class TestCodexCli: + @pytest.mark.covers("other.client_apps.codex_cli.tool_turn_streams") + def test_streams_a_tool_turn(self, client: ClientAppsClient, scoped_key: str, openai_model: str) -> None: + word = f"pong-{unique_marker()}" + events = unwrap_run( + client.run_codex( + key=scoped_key, + model=openai_model, + prompt=f"Run the shell command `echo {word}` and reply with exactly what it printed.", + ) + ) -@pytest.mark.covers("other.client_apps.codex_cli.tool_turn_streams") -def test_codex_cli_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str) -> None: - events = unwrap_run(client.run_codex(key=scoped_key, model=CHEAP_OPENAI_MODEL, prompt=PROMPT)) + completed_items = [event.item for event in events if isinstance(event, CodexItemCompleted)] + commands = [item for item in completed_items if isinstance(item, CodexCommandExecution)] + assert len(commands) == 1, events + assert commands[0].status == "completed", commands[0] + assert commands[0].exit_code == 0, commands[0] + assert word in commands[0].aggregated_output, commands[0] - completed_items = [event.item for event in events if isinstance(event, CodexItemCompleted)] - commands = [item for item in completed_items if isinstance(item, CodexCommandExecution)] - assert len(commands) == 1, events - assert commands[0].status == "completed", commands[0] - assert commands[0].exit_code == 0, commands[0] - assert "pong" in commands[0].aggregated_output, commands[0] + messages = [item for item in completed_items if isinstance(item, CodexAgentMessage)] + assert len(messages) == 1, events + assert word in messages[0].text, messages[0] - messages = [item for item in completed_items if isinstance(item, CodexAgentMessage)] - assert len(messages) == 1, events - assert "pong" in messages[0].text, messages[0] + turns = [event for event in events if isinstance(event, CodexTurnCompleted)] + assert len(turns) == 1, events + assert turns[0].usage.output_tokens > 0, turns[0] - turns = [event for event in events if isinstance(event, CodexTurnCompleted)] - assert len(turns) == 1, events - assert turns[0].usage.output_tokens > 0, turns[0] - - streamed_tool_turn_rows(client.proxy, scoped_key) + streamed_tool_turn_rows(client.proxy, scoped_key) diff --git a/tests/e2e/client_apps/test_vercel_ai_sdk_e2e.py b/tests/e2e/client_apps/test_vercel_ai_sdk_e2e.py index 06ae33fbf81..efc5fa31846 100644 --- a/tests/e2e/client_apps/test_vercel_ai_sdk_e2e.py +++ b/tests/e2e/client_apps/test_vercel_ai_sdk_e2e.py @@ -12,41 +12,44 @@ import pytest from client_apps_client import ClientAppsClient, unwrap_run from client_apps_models import VercelApi -from e2e_config import CHEAP_OPENAI_MODEL, unique_marker +from e2e_config import unique_marker from spend_rows import recorded_request, streamed_tool_turn_rows pytestmark = pytest.mark.e2e -@pytest.mark.parametrize( - "api", - [ - pytest.param( - "responses", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_responses.tool_turn_streams") - ), - pytest.param( - "chat", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_chat_completions.tool_turn_streams") - ), - ], -) -def test_vercel_ai_sdk_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str, api: VercelApi) -> None: - secret_word = f"pong-{unique_marker()}" - turn = unwrap_run( - client.run_vercel_ai_sdk(key=scoped_key, model=CHEAP_OPENAI_MODEL, api=api, secret_word=secret_word) +class TestVercelAiSdk: + @pytest.mark.parametrize( + "api", + [ + pytest.param( + "responses", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_responses.tool_turn_streams") + ), + pytest.param( + "chat", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_chat_completions.tool_turn_streams") + ), + ], ) + def test_streams_a_tool_turn( + self, client: ClientAppsClient, scoped_key: str, openai_model: str, api: VercelApi + ) -> None: + secret_word = f"pong-{unique_marker()}" + turn = unwrap_run( + client.run_vercel_ai_sdk(key=scoped_key, model=openai_model, api=api, secret_word=secret_word) + ) - assert turn.errors == [], turn - assert [call.tool_name for call in turn.tool_calls] == ["reveal_secret_word"], turn - assert [result.output.word for result in turn.tool_results] == [secret_word], turn - assert secret_word in turn.text, turn - assert turn.text_deltas >= 2, turn - assert turn.steps == 2, turn - assert turn.finish_reason == "stop", turn + assert turn.errors == [], turn + assert [call.tool_name for call in turn.tool_calls] == ["reveal_secret_word"], turn + assert [result.output.word for result in turn.tool_results] == [secret_word], turn + assert secret_word in turn.text, turn + assert turn.text_deltas >= 1, turn + assert turn.steps == 2, turn + assert turn.finish_reason == "stop", turn - rows = streamed_tool_turn_rows(client.proxy, scoped_key) - bodies = [recorded_request(row) for row in rows] - match api: - case "responses": - assert all(body.input is not None for body in bodies), rows - case "chat": - assert all(body.messages is not None for body in bodies), rows + rows = streamed_tool_turn_rows(client.proxy, scoped_key) + bodies = [recorded_request(row) for row in rows] + match api: + case "responses": + assert all(body.input is not None for body in bodies), rows + case "chat": + assert all(body.messages is not None for body in bodies), rows