test(e2e): register client-apps deployments per run and make the turns cache-proof

The suite now creates its own Anthropic and OpenAI deployments through
/model/new under run-unique names and deletes them at session end, so it no
longer depends on which aliases the proxy config serves. The names start
with the real model id because Claude Code 2.1.278 picks its request shape
from the model name and sends an unrecognized one a system-role probe that
Anthropic rejects with a 400 before the CLI falls back, which the proxy bills
as a failure row.

The echoed word is unique per run: the CI stack caches responses, and an
identical prompt across the three CI passes was served a $0 cache-hit row.
Tests move into the classes the e2e harness requires, the spend helper
asserts the row count it polls for, and the AI SDK cell accepts a single
text delta since the provider chunks the final sentence as it likes.
This commit is contained in:
mateo-berri 2026-09-22 11:03:30 -07:00
parent c45f63216f
commit 348ecf6556
5 changed files with 148 additions and 94 deletions

View file

@ -1,18 +1,56 @@
"""`client_apps` suite's `client` fixture.
"""`client_apps` suite's `client` fixture and the two deployments it drives.
Lifecycle (resources/scoped_key), proxy liveness gate, and the e2e/covers
markers all live in the parent tests/e2e/conftest.py. ClientAppsClient holds the
shared ProxyClient so the keys these tests mint tear down through it.
The suite registers its own deployments through /model/new, one Anthropic and
one OpenAI, under names unique to the run, and deletes them at session end, so
it never depends on which aliases the proxy's config file serves. Each carries
the provider key from the test runner's env when set; otherwise an os.environ
reference the proxy resolves from its own env. They stay on the real provider
path (`provider_live`) because a coding client's prompts change every run, so a
recorded response could never stand in for them. Each deployment's name starts
with the real model id: Claude Code 2.1.278 keys its request shape on the model
name, and an unrecognized one gets a system-role message Anthropic rejects with a
400 before the CLI falls back, which the proxy would bill as a failure row.
"""
from __future__ import annotations
import os
from collections.abc import Iterator
from typing import Final
import pytest
from client_apps_client import ClientAppsClient
from e2e_config import unique_marker
from models import LiteLLMParamsBody
from proxy_client import ProxyClient
@pytest.fixture(scope="session")
def client(proxy: ProxyClient) -> ClientAppsClient:
return ClientAppsClient(proxy=proxy)
@pytest.fixture(scope="session")
def anthropic_model(proxy: ProxyClient) -> Iterator[str]:
yield from _suite_model(proxy, "anthropic/claude-haiku-4-5", "ANTHROPIC_API_KEY")
@pytest.fixture(scope="session")
def openai_model(proxy: ProxyClient) -> Iterator[str]:
yield from _suite_model(proxy, "openai/gpt-5.5", "OPENAI_API_KEY")
def _suite_model(proxy: ProxyClient, provider_model: str, env_var: str) -> Iterator[str]:
model_name: Final = f"{provider_model.split('/', 1)[1]}-client-apps-{unique_marker()}"
model_id: Final = proxy.create_model(
model_name,
LiteLLMParamsBody(model=provider_model, api_key=os.environ.get(env_var) or f"os.environ/{env_var}"),
provider_live=True,
)
yield model_name
proxy.delete_model(model_id)

View file

@ -15,6 +15,7 @@ def recorded_request(row: SpendLogRow) -> RecordedRequest:
def streamed_tool_turn_rows(proxy: ProxyClient, key: str) -> list[SpendLogRow]:
rows = proxy.poll_logs_for_key(key, min_rows=2)
assert len(rows) >= 2, rows
assert all(row.status == "success" for row in rows), rows
assert all(row.spend is not None and row.spend > 0 for row in rows), rows
assert all(recorded_request(row).stream for row in rows), rows

View file

@ -2,12 +2,14 @@
turn against the proxy on a virtual key: it streams over `/v1/messages`, calls
the Bash tool, reads the tool result, and answers. Client side the stream-json
events carry the tool_use, the tool_result, the streamed deltas, and the final
result; proxy side the spend rows carry real spend and `stream: true`."""
result; proxy side the spend rows carry real spend and `stream: true`. The
echoed word is unique per run: the CI stack caches responses, and an
identical prompt across runs would be served a $0 cache-hit row instead of a
real call."""
from __future__ import annotations
import json
from typing import Final
import pytest
@ -21,54 +23,57 @@ from client_apps_models import (
ToolResultBlock,
ToolUseBlock,
)
from e2e_config import CHEAP_ANTHROPIC_MODEL
from e2e_config import unique_marker
from spend_rows import streamed_tool_turn_rows
pytestmark = pytest.mark.e2e
PROMPT: Final = "Use the Bash tool to run the command `echo pong` and report what it printed."
@pytest.mark.covers("other.client_apps.claude_code_cli.tool_turn_streams")
def test_claude_code_cli_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str) -> None:
events = unwrap_run(
client.run_claude_code(
key=scoped_key, model=CHEAP_ANTHROPIC_MODEL, prompt=PROMPT, allowed_tool="Bash(echo pong)"
class TestClaudeCodeCli:
@pytest.mark.covers("other.client_apps.claude_code_cli.tool_turn_streams")
def test_streams_a_tool_turn(self, client: ClientAppsClient, scoped_key: str, anthropic_model: str) -> None:
word = f"pong-{unique_marker()}"
events = unwrap_run(
client.run_claude_code(
key=scoped_key,
model=anthropic_model,
prompt=f"Use the Bash tool to run the command `echo {word}` and report what it printed.",
allowed_tool=f"Bash(echo {word})",
)
)
)
tool_uses = [
block
for event in events
if isinstance(event, ClaudeAssistantEvent) and isinstance(event.message.content, list)
for block in event.message.content
if isinstance(block, ToolUseBlock)
]
assert [block.name for block in tool_uses] == ["Bash"], events
tool_uses = [
block
for event in events
if isinstance(event, ClaudeAssistantEvent) and isinstance(event.message.content, list)
for block in event.message.content
if isinstance(block, ToolUseBlock)
]
assert [block.name for block in tool_uses] == ["Bash"], events
tool_results = [
block
for event in events
if isinstance(event, ClaudeUserEvent) and isinstance(event.message.content, list)
for block in event.message.content
if isinstance(block, ToolResultBlock)
]
assert len(tool_results) == 1, events
assert not tool_results[0].is_error, tool_results[0]
assert "pong" in json.dumps(tool_results[0].content), tool_results[0]
tool_results = [
block
for event in events
if isinstance(event, ClaudeUserEvent) and isinstance(event.message.content, list)
for block in event.message.content
if isinstance(block, ToolResultBlock)
]
assert len(tool_results) == 1, events
assert not tool_results[0].is_error, tool_results[0]
assert word in json.dumps(tool_results[0].content), tool_results[0]
delta_types = [
event.event.delta.type
for event in events
if isinstance(event, ClaudeStreamEvent) and isinstance(event.event, ContentBlockDelta)
]
assert "input_json_delta" in delta_types, delta_types
assert "text_delta" in delta_types, delta_types
delta_types = [
event.event.delta.type
for event in events
if isinstance(event, ClaudeStreamEvent) and isinstance(event.event, ContentBlockDelta)
]
assert "input_json_delta" in delta_types, delta_types
assert "text_delta" in delta_types, delta_types
results = [event for event in events if isinstance(event, ClaudeResultEvent)]
assert len(results) == 1, events
assert results[0].subtype == "success", results[0]
assert not results[0].is_error, results[0]
assert results[0].result is not None and "pong" in results[0].result, results[0]
results = [event for event in events if isinstance(event, ClaudeResultEvent)]
assert len(results) == 1, events
assert results[0].subtype == "success", results[0]
assert not results[0].is_error, results[0]
assert results[0].result is not None and word in results[0].result, results[0]
streamed_tool_turn_rows(client.proxy, scoped_key)
streamed_tool_turn_rows(client.proxy, scoped_key)

View file

@ -4,6 +4,9 @@ model provider (`wire_api = "responses"`, so `/v1/responses`): it runs a shell
command, reads its output, and answers. Client side the JSONL events carry the
completed command execution and the agent message; proxy side the spend rows
carry real spend and `stream: true` (Codex always streams the Responses API).
The echoed word is unique per run: the CI stack caches responses, and an
identical prompt across runs would be served a $0 cache-hit row instead of a
real call.
The throwaway config turns Codex's sandbox off (`sandbox_mode =
"danger-full-access"`): its Linux sandbox is bubblewrap, which cannot set up
@ -14,37 +17,41 @@ of value is reachable without the sandbox."""
from __future__ import annotations
from typing import Final
import pytest
from client_apps_client import ClientAppsClient, unwrap_run
from client_apps_models import CodexAgentMessage, CodexCommandExecution, CodexItemCompleted, CodexTurnCompleted
from e2e_config import CHEAP_OPENAI_MODEL
from e2e_config import unique_marker
from spend_rows import streamed_tool_turn_rows
pytestmark = pytest.mark.e2e
PROMPT: Final = "Run the shell command `echo pong` and reply with exactly what it printed."
class TestCodexCli:
@pytest.mark.covers("other.client_apps.codex_cli.tool_turn_streams")
def test_streams_a_tool_turn(self, client: ClientAppsClient, scoped_key: str, openai_model: str) -> None:
word = f"pong-{unique_marker()}"
events = unwrap_run(
client.run_codex(
key=scoped_key,
model=openai_model,
prompt=f"Run the shell command `echo {word}` and reply with exactly what it printed.",
)
)
@pytest.mark.covers("other.client_apps.codex_cli.tool_turn_streams")
def test_codex_cli_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str) -> None:
events = unwrap_run(client.run_codex(key=scoped_key, model=CHEAP_OPENAI_MODEL, prompt=PROMPT))
completed_items = [event.item for event in events if isinstance(event, CodexItemCompleted)]
commands = [item for item in completed_items if isinstance(item, CodexCommandExecution)]
assert len(commands) == 1, events
assert commands[0].status == "completed", commands[0]
assert commands[0].exit_code == 0, commands[0]
assert word in commands[0].aggregated_output, commands[0]
completed_items = [event.item for event in events if isinstance(event, CodexItemCompleted)]
commands = [item for item in completed_items if isinstance(item, CodexCommandExecution)]
assert len(commands) == 1, events
assert commands[0].status == "completed", commands[0]
assert commands[0].exit_code == 0, commands[0]
assert "pong" in commands[0].aggregated_output, commands[0]
messages = [item for item in completed_items if isinstance(item, CodexAgentMessage)]
assert len(messages) == 1, events
assert word in messages[0].text, messages[0]
messages = [item for item in completed_items if isinstance(item, CodexAgentMessage)]
assert len(messages) == 1, events
assert "pong" in messages[0].text, messages[0]
turns = [event for event in events if isinstance(event, CodexTurnCompleted)]
assert len(turns) == 1, events
assert turns[0].usage.output_tokens > 0, turns[0]
turns = [event for event in events if isinstance(event, CodexTurnCompleted)]
assert len(turns) == 1, events
assert turns[0].usage.output_tokens > 0, turns[0]
streamed_tool_turn_rows(client.proxy, scoped_key)
streamed_tool_turn_rows(client.proxy, scoped_key)

View file

@ -12,41 +12,44 @@ import pytest
from client_apps_client import ClientAppsClient, unwrap_run
from client_apps_models import VercelApi
from e2e_config import CHEAP_OPENAI_MODEL, unique_marker
from e2e_config import unique_marker
from spend_rows import recorded_request, streamed_tool_turn_rows
pytestmark = pytest.mark.e2e
@pytest.mark.parametrize(
"api",
[
pytest.param(
"responses", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_responses.tool_turn_streams")
),
pytest.param(
"chat", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_chat_completions.tool_turn_streams")
),
],
)
def test_vercel_ai_sdk_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str, api: VercelApi) -> None:
secret_word = f"pong-{unique_marker()}"
turn = unwrap_run(
client.run_vercel_ai_sdk(key=scoped_key, model=CHEAP_OPENAI_MODEL, api=api, secret_word=secret_word)
class TestVercelAiSdk:
@pytest.mark.parametrize(
"api",
[
pytest.param(
"responses", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_responses.tool_turn_streams")
),
pytest.param(
"chat", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_chat_completions.tool_turn_streams")
),
],
)
def test_streams_a_tool_turn(
self, client: ClientAppsClient, scoped_key: str, openai_model: str, api: VercelApi
) -> None:
secret_word = f"pong-{unique_marker()}"
turn = unwrap_run(
client.run_vercel_ai_sdk(key=scoped_key, model=openai_model, api=api, secret_word=secret_word)
)
assert turn.errors == [], turn
assert [call.tool_name for call in turn.tool_calls] == ["reveal_secret_word"], turn
assert [result.output.word for result in turn.tool_results] == [secret_word], turn
assert secret_word in turn.text, turn
assert turn.text_deltas >= 2, turn
assert turn.steps == 2, turn
assert turn.finish_reason == "stop", turn
assert turn.errors == [], turn
assert [call.tool_name for call in turn.tool_calls] == ["reveal_secret_word"], turn
assert [result.output.word for result in turn.tool_results] == [secret_word], turn
assert secret_word in turn.text, turn
assert turn.text_deltas >= 1, turn
assert turn.steps == 2, turn
assert turn.finish_reason == "stop", turn
rows = streamed_tool_turn_rows(client.proxy, scoped_key)
bodies = [recorded_request(row) for row in rows]
match api:
case "responses":
assert all(body.input is not None for body in bodies), rows
case "chat":
assert all(body.messages is not None for body in bodies), rows
rows = streamed_tool_turn_rows(client.proxy, scoped_key)
bodies = [recorded_request(row) for row in rows]
match api:
case "responses":
assert all(body.input is not None for body in bodies), rows
case "chat":
assert all(body.messages is not None for body in bodies), rows