diff --git a/tests/e2e/client_apps/conftest.py b/tests/e2e/client_apps/conftest.py index d35e8d7a597..b68945700bb 100644 --- a/tests/e2e/client_apps/conftest.py +++ b/tests/e2e/client_apps/conftest.py @@ -1,18 +1,56 @@ -"""`client_apps` suite's `client` fixture. +"""`client_apps` suite's `client` fixture and the two deployments it drives. Lifecycle (resources/scoped_key), proxy liveness gate, and the e2e/covers markers all live in the parent tests/e2e/conftest.py. ClientAppsClient holds the shared ProxyClient so the keys these tests mint tear down through it. + +The suite registers its own deployments through /model/new, one Anthropic and +one OpenAI, under names unique to the run, and deletes them at session end, so +it never depends on which aliases the proxy's config file serves. Each carries +the provider key from the test runner's env when set; otherwise an os.environ +reference the proxy resolves from its own env. They stay on the real provider +path (`provider_live`) because a coding client's prompts change every run, so a +recorded response could never stand in for them. Each deployment's name starts +with the real model id: Claude Code 2.1.278 keys its request shape on the model +name, and an unrecognized one gets a system-role message Anthropic rejects with a +400 before the CLI falls back, which the proxy would bill as a failure row. """ from __future__ import annotations +import os +from collections.abc import Iterator +from typing import Final + import pytest from client_apps_client import ClientAppsClient +from e2e_config import unique_marker +from models import LiteLLMParamsBody from proxy_client import ProxyClient @pytest.fixture(scope="session") def client(proxy: ProxyClient) -> ClientAppsClient: return ClientAppsClient(proxy=proxy) + + +@pytest.fixture(scope="session") +def anthropic_model(proxy: ProxyClient) -> Iterator[str]: + yield from _suite_model(proxy, "anthropic/claude-haiku-4-5", "ANTHROPIC_API_KEY") + + +@pytest.fixture(scope="session") +def openai_model(proxy: ProxyClient) -> Iterator[str]: + yield from _suite_model(proxy, "openai/gpt-5.5", "OPENAI_API_KEY") + + +def _suite_model(proxy: ProxyClient, provider_model: str, env_var: str) -> Iterator[str]: + model_name: Final = f"{provider_model.split('/', 1)[1]}-client-apps-{unique_marker()}" + model_id: Final = proxy.create_model( + model_name, + LiteLLMParamsBody(model=provider_model, api_key=os.environ.get(env_var) or f"os.environ/{env_var}"), + provider_live=True, + ) + yield model_name + proxy.delete_model(model_id) diff --git a/tests/e2e/client_apps/spend_rows.py b/tests/e2e/client_apps/spend_rows.py index 282b4250d57..05329d6e0b9 100644 --- a/tests/e2e/client_apps/spend_rows.py +++ b/tests/e2e/client_apps/spend_rows.py @@ -15,6 +15,7 @@ def recorded_request(row: SpendLogRow) -> RecordedRequest: def streamed_tool_turn_rows(proxy: ProxyClient, key: str) -> list[SpendLogRow]: rows = proxy.poll_logs_for_key(key, min_rows=2) + assert len(rows) >= 2, rows assert all(row.status == "success" for row in rows), rows assert all(row.spend is not None and row.spend > 0 for row in rows), rows assert all(recorded_request(row).stream for row in rows), rows diff --git a/tests/e2e/client_apps/test_claude_code_cli_e2e.py b/tests/e2e/client_apps/test_claude_code_cli_e2e.py index b3de2ea2a09..62c04d7c05a 100644 --- a/tests/e2e/client_apps/test_claude_code_cli_e2e.py +++ b/tests/e2e/client_apps/test_claude_code_cli_e2e.py @@ -2,12 +2,14 @@ turn against the proxy on a virtual key: it streams over `/v1/messages`, calls the Bash tool, reads the tool result, and answers. Client side the stream-json events carry the tool_use, the tool_result, the streamed deltas, and the final -result; proxy side the spend rows carry real spend and `stream: true`.""" +result; proxy side the spend rows carry real spend and `stream: true`. The +echoed word is unique per run: the CI stack caches responses, and an +identical prompt across runs would be served a $0 cache-hit row instead of a +real call.""" from __future__ import annotations import json -from typing import Final import pytest @@ -21,54 +23,57 @@ from client_apps_models import ( ToolResultBlock, ToolUseBlock, ) -from e2e_config import CHEAP_ANTHROPIC_MODEL +from e2e_config import unique_marker from spend_rows import streamed_tool_turn_rows pytestmark = pytest.mark.e2e -PROMPT: Final = "Use the Bash tool to run the command `echo pong` and report what it printed." - -@pytest.mark.covers("other.client_apps.claude_code_cli.tool_turn_streams") -def test_claude_code_cli_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str) -> None: - events = unwrap_run( - client.run_claude_code( - key=scoped_key, model=CHEAP_ANTHROPIC_MODEL, prompt=PROMPT, allowed_tool="Bash(echo pong)" +class TestClaudeCodeCli: + @pytest.mark.covers("other.client_apps.claude_code_cli.tool_turn_streams") + def test_streams_a_tool_turn(self, client: ClientAppsClient, scoped_key: str, anthropic_model: str) -> None: + word = f"pong-{unique_marker()}" + events = unwrap_run( + client.run_claude_code( + key=scoped_key, + model=anthropic_model, + prompt=f"Use the Bash tool to run the command `echo {word}` and report what it printed.", + allowed_tool=f"Bash(echo {word})", + ) ) - ) - tool_uses = [ - block - for event in events - if isinstance(event, ClaudeAssistantEvent) and isinstance(event.message.content, list) - for block in event.message.content - if isinstance(block, ToolUseBlock) - ] - assert [block.name for block in tool_uses] == ["Bash"], events + tool_uses = [ + block + for event in events + if isinstance(event, ClaudeAssistantEvent) and isinstance(event.message.content, list) + for block in event.message.content + if isinstance(block, ToolUseBlock) + ] + assert [block.name for block in tool_uses] == ["Bash"], events - tool_results = [ - block - for event in events - if isinstance(event, ClaudeUserEvent) and isinstance(event.message.content, list) - for block in event.message.content - if isinstance(block, ToolResultBlock) - ] - assert len(tool_results) == 1, events - assert not tool_results[0].is_error, tool_results[0] - assert "pong" in json.dumps(tool_results[0].content), tool_results[0] + tool_results = [ + block + for event in events + if isinstance(event, ClaudeUserEvent) and isinstance(event.message.content, list) + for block in event.message.content + if isinstance(block, ToolResultBlock) + ] + assert len(tool_results) == 1, events + assert not tool_results[0].is_error, tool_results[0] + assert word in json.dumps(tool_results[0].content), tool_results[0] - delta_types = [ - event.event.delta.type - for event in events - if isinstance(event, ClaudeStreamEvent) and isinstance(event.event, ContentBlockDelta) - ] - assert "input_json_delta" in delta_types, delta_types - assert "text_delta" in delta_types, delta_types + delta_types = [ + event.event.delta.type + for event in events + if isinstance(event, ClaudeStreamEvent) and isinstance(event.event, ContentBlockDelta) + ] + assert "input_json_delta" in delta_types, delta_types + assert "text_delta" in delta_types, delta_types - results = [event for event in events if isinstance(event, ClaudeResultEvent)] - assert len(results) == 1, events - assert results[0].subtype == "success", results[0] - assert not results[0].is_error, results[0] - assert results[0].result is not None and "pong" in results[0].result, results[0] + results = [event for event in events if isinstance(event, ClaudeResultEvent)] + assert len(results) == 1, events + assert results[0].subtype == "success", results[0] + assert not results[0].is_error, results[0] + assert results[0].result is not None and word in results[0].result, results[0] - streamed_tool_turn_rows(client.proxy, scoped_key) + streamed_tool_turn_rows(client.proxy, scoped_key) diff --git a/tests/e2e/client_apps/test_codex_cli_e2e.py b/tests/e2e/client_apps/test_codex_cli_e2e.py index 53ed2c4f686..5bcdadb0a43 100644 --- a/tests/e2e/client_apps/test_codex_cli_e2e.py +++ b/tests/e2e/client_apps/test_codex_cli_e2e.py @@ -4,6 +4,9 @@ model provider (`wire_api = "responses"`, so `/v1/responses`): it runs a shell command, reads its output, and answers. Client side the JSONL events carry the completed command execution and the agent message; proxy side the spend rows carry real spend and `stream: true` (Codex always streams the Responses API). +The echoed word is unique per run: the CI stack caches responses, and an +identical prompt across runs would be served a $0 cache-hit row instead of a +real call. The throwaway config turns Codex's sandbox off (`sandbox_mode = "danger-full-access"`): its Linux sandbox is bubblewrap, which cannot set up @@ -14,37 +17,41 @@ of value is reachable without the sandbox.""" from __future__ import annotations -from typing import Final - import pytest from client_apps_client import ClientAppsClient, unwrap_run from client_apps_models import CodexAgentMessage, CodexCommandExecution, CodexItemCompleted, CodexTurnCompleted -from e2e_config import CHEAP_OPENAI_MODEL +from e2e_config import unique_marker from spend_rows import streamed_tool_turn_rows pytestmark = pytest.mark.e2e -PROMPT: Final = "Run the shell command `echo pong` and reply with exactly what it printed." +class TestCodexCli: + @pytest.mark.covers("other.client_apps.codex_cli.tool_turn_streams") + def test_streams_a_tool_turn(self, client: ClientAppsClient, scoped_key: str, openai_model: str) -> None: + word = f"pong-{unique_marker()}" + events = unwrap_run( + client.run_codex( + key=scoped_key, + model=openai_model, + prompt=f"Run the shell command `echo {word}` and reply with exactly what it printed.", + ) + ) -@pytest.mark.covers("other.client_apps.codex_cli.tool_turn_streams") -def test_codex_cli_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str) -> None: - events = unwrap_run(client.run_codex(key=scoped_key, model=CHEAP_OPENAI_MODEL, prompt=PROMPT)) + completed_items = [event.item for event in events if isinstance(event, CodexItemCompleted)] + commands = [item for item in completed_items if isinstance(item, CodexCommandExecution)] + assert len(commands) == 1, events + assert commands[0].status == "completed", commands[0] + assert commands[0].exit_code == 0, commands[0] + assert word in commands[0].aggregated_output, commands[0] - completed_items = [event.item for event in events if isinstance(event, CodexItemCompleted)] - commands = [item for item in completed_items if isinstance(item, CodexCommandExecution)] - assert len(commands) == 1, events - assert commands[0].status == "completed", commands[0] - assert commands[0].exit_code == 0, commands[0] - assert "pong" in commands[0].aggregated_output, commands[0] + messages = [item for item in completed_items if isinstance(item, CodexAgentMessage)] + assert len(messages) == 1, events + assert word in messages[0].text, messages[0] - messages = [item for item in completed_items if isinstance(item, CodexAgentMessage)] - assert len(messages) == 1, events - assert "pong" in messages[0].text, messages[0] + turns = [event for event in events if isinstance(event, CodexTurnCompleted)] + assert len(turns) == 1, events + assert turns[0].usage.output_tokens > 0, turns[0] - turns = [event for event in events if isinstance(event, CodexTurnCompleted)] - assert len(turns) == 1, events - assert turns[0].usage.output_tokens > 0, turns[0] - - streamed_tool_turn_rows(client.proxy, scoped_key) + streamed_tool_turn_rows(client.proxy, scoped_key) diff --git a/tests/e2e/client_apps/test_vercel_ai_sdk_e2e.py b/tests/e2e/client_apps/test_vercel_ai_sdk_e2e.py index 06ae33fbf81..efc5fa31846 100644 --- a/tests/e2e/client_apps/test_vercel_ai_sdk_e2e.py +++ b/tests/e2e/client_apps/test_vercel_ai_sdk_e2e.py @@ -12,41 +12,44 @@ import pytest from client_apps_client import ClientAppsClient, unwrap_run from client_apps_models import VercelApi -from e2e_config import CHEAP_OPENAI_MODEL, unique_marker +from e2e_config import unique_marker from spend_rows import recorded_request, streamed_tool_turn_rows pytestmark = pytest.mark.e2e -@pytest.mark.parametrize( - "api", - [ - pytest.param( - "responses", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_responses.tool_turn_streams") - ), - pytest.param( - "chat", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_chat_completions.tool_turn_streams") - ), - ], -) -def test_vercel_ai_sdk_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str, api: VercelApi) -> None: - secret_word = f"pong-{unique_marker()}" - turn = unwrap_run( - client.run_vercel_ai_sdk(key=scoped_key, model=CHEAP_OPENAI_MODEL, api=api, secret_word=secret_word) +class TestVercelAiSdk: + @pytest.mark.parametrize( + "api", + [ + pytest.param( + "responses", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_responses.tool_turn_streams") + ), + pytest.param( + "chat", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_chat_completions.tool_turn_streams") + ), + ], ) + def test_streams_a_tool_turn( + self, client: ClientAppsClient, scoped_key: str, openai_model: str, api: VercelApi + ) -> None: + secret_word = f"pong-{unique_marker()}" + turn = unwrap_run( + client.run_vercel_ai_sdk(key=scoped_key, model=openai_model, api=api, secret_word=secret_word) + ) - assert turn.errors == [], turn - assert [call.tool_name for call in turn.tool_calls] == ["reveal_secret_word"], turn - assert [result.output.word for result in turn.tool_results] == [secret_word], turn - assert secret_word in turn.text, turn - assert turn.text_deltas >= 2, turn - assert turn.steps == 2, turn - assert turn.finish_reason == "stop", turn + assert turn.errors == [], turn + assert [call.tool_name for call in turn.tool_calls] == ["reveal_secret_word"], turn + assert [result.output.word for result in turn.tool_results] == [secret_word], turn + assert secret_word in turn.text, turn + assert turn.text_deltas >= 1, turn + assert turn.steps == 2, turn + assert turn.finish_reason == "stop", turn - rows = streamed_tool_turn_rows(client.proxy, scoped_key) - bodies = [recorded_request(row) for row in rows] - match api: - case "responses": - assert all(body.input is not None for body in bodies), rows - case "chat": - assert all(body.messages is not None for body in bodies), rows + rows = streamed_tool_turn_rows(client.proxy, scoped_key) + bodies = [recorded_request(row) for row in rows] + match api: + case "responses": + assert all(body.input is not None for body in bodies), rows + case "chat": + assert all(body.messages is not None for body in bodies), rows