mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
test(e2e): register client-apps deployments per run and make the turns cache-proof
The suite now creates its own Anthropic and OpenAI deployments through /model/new under run-unique names and deletes them at session end, so it no longer depends on which aliases the proxy config serves. The names start with the real model id because Claude Code 2.1.278 picks its request shape from the model name and sends an unrecognized one a system-role probe that Anthropic rejects with a 400 before the CLI falls back, which the proxy bills as a failure row. The echoed word is unique per run: the CI stack caches responses, and an identical prompt across the three CI passes was served a $0 cache-hit row. Tests move into the classes the e2e harness requires, the spend helper asserts the row count it polls for, and the AI SDK cell accepts a single text delta since the provider chunks the final sentence as it likes.
This commit is contained in:
parent
c45f63216f
commit
348ecf6556
5 changed files with 148 additions and 94 deletions
|
|
@ -1,18 +1,56 @@
|
|||
"""`client_apps` suite's `client` fixture.
|
||||
"""`client_apps` suite's `client` fixture and the two deployments it drives.
|
||||
|
||||
Lifecycle (resources/scoped_key), proxy liveness gate, and the e2e/covers
|
||||
markers all live in the parent tests/e2e/conftest.py. ClientAppsClient holds the
|
||||
shared ProxyClient so the keys these tests mint tear down through it.
|
||||
|
||||
The suite registers its own deployments through /model/new, one Anthropic and
|
||||
one OpenAI, under names unique to the run, and deletes them at session end, so
|
||||
it never depends on which aliases the proxy's config file serves. Each carries
|
||||
the provider key from the test runner's env when set; otherwise an os.environ
|
||||
reference the proxy resolves from its own env. They stay on the real provider
|
||||
path (`provider_live`) because a coding client's prompts change every run, so a
|
||||
recorded response could never stand in for them. Each deployment's name starts
|
||||
with the real model id: Claude Code 2.1.278 keys its request shape on the model
|
||||
name, and an unrecognized one gets a system-role message Anthropic rejects with a
|
||||
400 before the CLI falls back, which the proxy would bill as a failure row.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Iterator
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from client_apps_client import ClientAppsClient
|
||||
from e2e_config import unique_marker
|
||||
from models import LiteLLMParamsBody
|
||||
from proxy_client import ProxyClient
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def client(proxy: ProxyClient) -> ClientAppsClient:
|
||||
return ClientAppsClient(proxy=proxy)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def anthropic_model(proxy: ProxyClient) -> Iterator[str]:
|
||||
yield from _suite_model(proxy, "anthropic/claude-haiku-4-5", "ANTHROPIC_API_KEY")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def openai_model(proxy: ProxyClient) -> Iterator[str]:
|
||||
yield from _suite_model(proxy, "openai/gpt-5.5", "OPENAI_API_KEY")
|
||||
|
||||
|
||||
def _suite_model(proxy: ProxyClient, provider_model: str, env_var: str) -> Iterator[str]:
|
||||
model_name: Final = f"{provider_model.split('/', 1)[1]}-client-apps-{unique_marker()}"
|
||||
model_id: Final = proxy.create_model(
|
||||
model_name,
|
||||
LiteLLMParamsBody(model=provider_model, api_key=os.environ.get(env_var) or f"os.environ/{env_var}"),
|
||||
provider_live=True,
|
||||
)
|
||||
yield model_name
|
||||
proxy.delete_model(model_id)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ def recorded_request(row: SpendLogRow) -> RecordedRequest:
|
|||
|
||||
def streamed_tool_turn_rows(proxy: ProxyClient, key: str) -> list[SpendLogRow]:
|
||||
rows = proxy.poll_logs_for_key(key, min_rows=2)
|
||||
assert len(rows) >= 2, rows
|
||||
assert all(row.status == "success" for row in rows), rows
|
||||
assert all(row.spend is not None and row.spend > 0 for row in rows), rows
|
||||
assert all(recorded_request(row).stream for row in rows), rows
|
||||
|
|
|
|||
|
|
@ -2,12 +2,14 @@
|
|||
turn against the proxy on a virtual key: it streams over `/v1/messages`, calls
|
||||
the Bash tool, reads the tool result, and answers. Client side the stream-json
|
||||
events carry the tool_use, the tool_result, the streamed deltas, and the final
|
||||
result; proxy side the spend rows carry real spend and `stream: true`."""
|
||||
result; proxy side the spend rows carry real spend and `stream: true`. The
|
||||
echoed word is unique per run: the CI stack caches responses, and an
|
||||
identical prompt across runs would be served a $0 cache-hit row instead of a
|
||||
real call."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -21,54 +23,57 @@ from client_apps_models import (
|
|||
ToolResultBlock,
|
||||
ToolUseBlock,
|
||||
)
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL
|
||||
from e2e_config import unique_marker
|
||||
from spend_rows import streamed_tool_turn_rows
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
PROMPT: Final = "Use the Bash tool to run the command `echo pong` and report what it printed."
|
||||
|
||||
|
||||
@pytest.mark.covers("other.client_apps.claude_code_cli.tool_turn_streams")
|
||||
def test_claude_code_cli_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str) -> None:
|
||||
events = unwrap_run(
|
||||
client.run_claude_code(
|
||||
key=scoped_key, model=CHEAP_ANTHROPIC_MODEL, prompt=PROMPT, allowed_tool="Bash(echo pong)"
|
||||
class TestClaudeCodeCli:
|
||||
@pytest.mark.covers("other.client_apps.claude_code_cli.tool_turn_streams")
|
||||
def test_streams_a_tool_turn(self, client: ClientAppsClient, scoped_key: str, anthropic_model: str) -> None:
|
||||
word = f"pong-{unique_marker()}"
|
||||
events = unwrap_run(
|
||||
client.run_claude_code(
|
||||
key=scoped_key,
|
||||
model=anthropic_model,
|
||||
prompt=f"Use the Bash tool to run the command `echo {word}` and report what it printed.",
|
||||
allowed_tool=f"Bash(echo {word})",
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
tool_uses = [
|
||||
block
|
||||
for event in events
|
||||
if isinstance(event, ClaudeAssistantEvent) and isinstance(event.message.content, list)
|
||||
for block in event.message.content
|
||||
if isinstance(block, ToolUseBlock)
|
||||
]
|
||||
assert [block.name for block in tool_uses] == ["Bash"], events
|
||||
tool_uses = [
|
||||
block
|
||||
for event in events
|
||||
if isinstance(event, ClaudeAssistantEvent) and isinstance(event.message.content, list)
|
||||
for block in event.message.content
|
||||
if isinstance(block, ToolUseBlock)
|
||||
]
|
||||
assert [block.name for block in tool_uses] == ["Bash"], events
|
||||
|
||||
tool_results = [
|
||||
block
|
||||
for event in events
|
||||
if isinstance(event, ClaudeUserEvent) and isinstance(event.message.content, list)
|
||||
for block in event.message.content
|
||||
if isinstance(block, ToolResultBlock)
|
||||
]
|
||||
assert len(tool_results) == 1, events
|
||||
assert not tool_results[0].is_error, tool_results[0]
|
||||
assert "pong" in json.dumps(tool_results[0].content), tool_results[0]
|
||||
tool_results = [
|
||||
block
|
||||
for event in events
|
||||
if isinstance(event, ClaudeUserEvent) and isinstance(event.message.content, list)
|
||||
for block in event.message.content
|
||||
if isinstance(block, ToolResultBlock)
|
||||
]
|
||||
assert len(tool_results) == 1, events
|
||||
assert not tool_results[0].is_error, tool_results[0]
|
||||
assert word in json.dumps(tool_results[0].content), tool_results[0]
|
||||
|
||||
delta_types = [
|
||||
event.event.delta.type
|
||||
for event in events
|
||||
if isinstance(event, ClaudeStreamEvent) and isinstance(event.event, ContentBlockDelta)
|
||||
]
|
||||
assert "input_json_delta" in delta_types, delta_types
|
||||
assert "text_delta" in delta_types, delta_types
|
||||
delta_types = [
|
||||
event.event.delta.type
|
||||
for event in events
|
||||
if isinstance(event, ClaudeStreamEvent) and isinstance(event.event, ContentBlockDelta)
|
||||
]
|
||||
assert "input_json_delta" in delta_types, delta_types
|
||||
assert "text_delta" in delta_types, delta_types
|
||||
|
||||
results = [event for event in events if isinstance(event, ClaudeResultEvent)]
|
||||
assert len(results) == 1, events
|
||||
assert results[0].subtype == "success", results[0]
|
||||
assert not results[0].is_error, results[0]
|
||||
assert results[0].result is not None and "pong" in results[0].result, results[0]
|
||||
results = [event for event in events if isinstance(event, ClaudeResultEvent)]
|
||||
assert len(results) == 1, events
|
||||
assert results[0].subtype == "success", results[0]
|
||||
assert not results[0].is_error, results[0]
|
||||
assert results[0].result is not None and word in results[0].result, results[0]
|
||||
|
||||
streamed_tool_turn_rows(client.proxy, scoped_key)
|
||||
streamed_tool_turn_rows(client.proxy, scoped_key)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,9 @@ model provider (`wire_api = "responses"`, so `/v1/responses`): it runs a shell
|
|||
command, reads its output, and answers. Client side the JSONL events carry the
|
||||
completed command execution and the agent message; proxy side the spend rows
|
||||
carry real spend and `stream: true` (Codex always streams the Responses API).
|
||||
The echoed word is unique per run: the CI stack caches responses, and an
|
||||
identical prompt across runs would be served a $0 cache-hit row instead of a
|
||||
real call.
|
||||
|
||||
The throwaway config turns Codex's sandbox off (`sandbox_mode =
|
||||
"danger-full-access"`): its Linux sandbox is bubblewrap, which cannot set up
|
||||
|
|
@ -14,37 +17,41 @@ of value is reachable without the sandbox."""
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from client_apps_client import ClientAppsClient, unwrap_run
|
||||
from client_apps_models import CodexAgentMessage, CodexCommandExecution, CodexItemCompleted, CodexTurnCompleted
|
||||
from e2e_config import CHEAP_OPENAI_MODEL
|
||||
from e2e_config import unique_marker
|
||||
from spend_rows import streamed_tool_turn_rows
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
PROMPT: Final = "Run the shell command `echo pong` and reply with exactly what it printed."
|
||||
|
||||
class TestCodexCli:
|
||||
@pytest.mark.covers("other.client_apps.codex_cli.tool_turn_streams")
|
||||
def test_streams_a_tool_turn(self, client: ClientAppsClient, scoped_key: str, openai_model: str) -> None:
|
||||
word = f"pong-{unique_marker()}"
|
||||
events = unwrap_run(
|
||||
client.run_codex(
|
||||
key=scoped_key,
|
||||
model=openai_model,
|
||||
prompt=f"Run the shell command `echo {word}` and reply with exactly what it printed.",
|
||||
)
|
||||
)
|
||||
|
||||
@pytest.mark.covers("other.client_apps.codex_cli.tool_turn_streams")
|
||||
def test_codex_cli_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str) -> None:
|
||||
events = unwrap_run(client.run_codex(key=scoped_key, model=CHEAP_OPENAI_MODEL, prompt=PROMPT))
|
||||
completed_items = [event.item for event in events if isinstance(event, CodexItemCompleted)]
|
||||
commands = [item for item in completed_items if isinstance(item, CodexCommandExecution)]
|
||||
assert len(commands) == 1, events
|
||||
assert commands[0].status == "completed", commands[0]
|
||||
assert commands[0].exit_code == 0, commands[0]
|
||||
assert word in commands[0].aggregated_output, commands[0]
|
||||
|
||||
completed_items = [event.item for event in events if isinstance(event, CodexItemCompleted)]
|
||||
commands = [item for item in completed_items if isinstance(item, CodexCommandExecution)]
|
||||
assert len(commands) == 1, events
|
||||
assert commands[0].status == "completed", commands[0]
|
||||
assert commands[0].exit_code == 0, commands[0]
|
||||
assert "pong" in commands[0].aggregated_output, commands[0]
|
||||
messages = [item for item in completed_items if isinstance(item, CodexAgentMessage)]
|
||||
assert len(messages) == 1, events
|
||||
assert word in messages[0].text, messages[0]
|
||||
|
||||
messages = [item for item in completed_items if isinstance(item, CodexAgentMessage)]
|
||||
assert len(messages) == 1, events
|
||||
assert "pong" in messages[0].text, messages[0]
|
||||
turns = [event for event in events if isinstance(event, CodexTurnCompleted)]
|
||||
assert len(turns) == 1, events
|
||||
assert turns[0].usage.output_tokens > 0, turns[0]
|
||||
|
||||
turns = [event for event in events if isinstance(event, CodexTurnCompleted)]
|
||||
assert len(turns) == 1, events
|
||||
assert turns[0].usage.output_tokens > 0, turns[0]
|
||||
|
||||
streamed_tool_turn_rows(client.proxy, scoped_key)
|
||||
streamed_tool_turn_rows(client.proxy, scoped_key)
|
||||
|
|
|
|||
|
|
@ -12,41 +12,44 @@ import pytest
|
|||
|
||||
from client_apps_client import ClientAppsClient, unwrap_run
|
||||
from client_apps_models import VercelApi
|
||||
from e2e_config import CHEAP_OPENAI_MODEL, unique_marker
|
||||
from e2e_config import unique_marker
|
||||
from spend_rows import recorded_request, streamed_tool_turn_rows
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api",
|
||||
[
|
||||
pytest.param(
|
||||
"responses", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_responses.tool_turn_streams")
|
||||
),
|
||||
pytest.param(
|
||||
"chat", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_chat_completions.tool_turn_streams")
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_vercel_ai_sdk_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str, api: VercelApi) -> None:
|
||||
secret_word = f"pong-{unique_marker()}"
|
||||
turn = unwrap_run(
|
||||
client.run_vercel_ai_sdk(key=scoped_key, model=CHEAP_OPENAI_MODEL, api=api, secret_word=secret_word)
|
||||
class TestVercelAiSdk:
|
||||
@pytest.mark.parametrize(
|
||||
"api",
|
||||
[
|
||||
pytest.param(
|
||||
"responses", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_responses.tool_turn_streams")
|
||||
),
|
||||
pytest.param(
|
||||
"chat", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_chat_completions.tool_turn_streams")
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_streams_a_tool_turn(
|
||||
self, client: ClientAppsClient, scoped_key: str, openai_model: str, api: VercelApi
|
||||
) -> None:
|
||||
secret_word = f"pong-{unique_marker()}"
|
||||
turn = unwrap_run(
|
||||
client.run_vercel_ai_sdk(key=scoped_key, model=openai_model, api=api, secret_word=secret_word)
|
||||
)
|
||||
|
||||
assert turn.errors == [], turn
|
||||
assert [call.tool_name for call in turn.tool_calls] == ["reveal_secret_word"], turn
|
||||
assert [result.output.word for result in turn.tool_results] == [secret_word], turn
|
||||
assert secret_word in turn.text, turn
|
||||
assert turn.text_deltas >= 2, turn
|
||||
assert turn.steps == 2, turn
|
||||
assert turn.finish_reason == "stop", turn
|
||||
assert turn.errors == [], turn
|
||||
assert [call.tool_name for call in turn.tool_calls] == ["reveal_secret_word"], turn
|
||||
assert [result.output.word for result in turn.tool_results] == [secret_word], turn
|
||||
assert secret_word in turn.text, turn
|
||||
assert turn.text_deltas >= 1, turn
|
||||
assert turn.steps == 2, turn
|
||||
assert turn.finish_reason == "stop", turn
|
||||
|
||||
rows = streamed_tool_turn_rows(client.proxy, scoped_key)
|
||||
bodies = [recorded_request(row) for row in rows]
|
||||
match api:
|
||||
case "responses":
|
||||
assert all(body.input is not None for body in bodies), rows
|
||||
case "chat":
|
||||
assert all(body.messages is not None for body in bodies), rows
|
||||
rows = streamed_tool_turn_rows(client.proxy, scoped_key)
|
||||
bodies = [recorded_request(row) for row in rows]
|
||||
match api:
|
||||
case "responses":
|
||||
assert all(body.input is not None for body in bodies), rows
|
||||
case "chat":
|
||||
assert all(body.messages is not None for body in bodies), rows
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue