From 173f4c45ed315b8358c0fbfa39ff5fa2bd5a054e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:01:11 -0700 Subject: [PATCH] test(e2e): drive the proxy through the Claude Code CLI, the Codex CLI, and the Vercel AI SDK --- .github/workflows/test-e2e-changed.yml | 11 + tests/e2e/AGENTS.md | 1 + tests/e2e/client_apps/.nvmrc | 1 + tests/e2e/client_apps/client_apps_client.py | 189 ++++++++ tests/e2e/client_apps/client_apps_models.py | 172 ++++++++ tests/e2e/client_apps/conftest.py | 18 + tests/e2e/client_apps/package-lock.json | 416 ++++++++++++++++++ tests/e2e/client_apps/package.json | 15 + tests/e2e/client_apps/spend_rows.py | 21 + .../client_apps/test_claude_code_cli_e2e.py | 74 ++++ tests/e2e/client_apps/test_codex_cli_e2e.py | 43 ++ .../e2e/client_apps/test_vercel_ai_sdk_e2e.py | 52 +++ tests/e2e/client_apps/vercel_ai_turn.mjs | 78 ++++ tests/e2e/coverage_registry/other.yaml | 5 + 14 files changed, 1096 insertions(+) create mode 100644 tests/e2e/client_apps/.nvmrc create mode 100644 tests/e2e/client_apps/client_apps_client.py create mode 100644 tests/e2e/client_apps/client_apps_models.py create mode 100644 tests/e2e/client_apps/conftest.py create mode 100644 tests/e2e/client_apps/package-lock.json create mode 100644 tests/e2e/client_apps/package.json create mode 100644 tests/e2e/client_apps/spend_rows.py create mode 100644 tests/e2e/client_apps/test_claude_code_cli_e2e.py create mode 100644 tests/e2e/client_apps/test_codex_cli_e2e.py create mode 100644 tests/e2e/client_apps/test_vercel_ai_sdk_e2e.py create mode 100644 tests/e2e/client_apps/vercel_ai_turn.mjs diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml index 8e03a902383..63a870c9ed3 100644 --- a/.github/workflows/test-e2e-changed.yml +++ b/.github/workflows/test-e2e-changed.yml @@ -129,6 +129,17 @@ jobs: - name: Install Playwright chromium run: uv run --no-sync playwright install --with-deps chromium + - name: Set up Node for the client_apps suite + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + with: + node-version-file: tests/e2e/client_apps/.nvmrc + cache: npm + cache-dependency-path: tests/e2e/client_apps/package-lock.json + + - name: Install the client apps the client_apps suite drives + working-directory: tests/e2e/client_apps + run: npm ci --no-audit --no-fund + - name: Configure AWS credentials id: aws uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index b00b7dfac95..6ba4359ecb9 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -48,6 +48,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke +- `client_apps/` - real client applications driven headlessly against the proxy on a virtual key, one streamed turn with a tool call each: the Claude Code CLI (`/v1/messages`), the Codex CLI (`/v1/responses`), and a Vercel AI SDK script (`vercel_ai_turn.mjs`, over both `/v1/responses` and `/v1/chat/completions`). The CLIs and the SDK are pinned npm dependencies of the suite's own `package.json` (`npm ci` in this folder before running; `test-e2e-changed.yml` does it in CI), the Claude Code path reuses `claude_code/cli_driver.py`, and every `.py` file here follows the Python rules in this file - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` ## MCP suite: real Datadog only diff --git a/tests/e2e/client_apps/.nvmrc b/tests/e2e/client_apps/.nvmrc new file mode 100644 index 00000000000..60ade1ae01e --- /dev/null +++ b/tests/e2e/client_apps/.nvmrc @@ -0,0 +1 @@ +24.19.0 diff --git a/tests/e2e/client_apps/client_apps_client.py b/tests/e2e/client_apps/client_apps_client.py new file mode 100644 index 00000000000..a14b1198451 --- /dev/null +++ b/tests/e2e/client_apps/client_apps_client.py @@ -0,0 +1,189 @@ +"""Client for the `client_apps` suite: runs the Claude Code CLI and the Codex CLI +installed into this folder by `npm ci`, and the Vercel AI SDK script +`vercel_ai_turn.mjs`, headlessly against the proxy on a virtual key. Each run gets +an empty HOME and only an allowlisted environment, reads nothing from stdin, and +comes back as the typed events or summary it printed, or a `CommandFailed` value. + +Holds the shared ProxyClient so `resources` / `scoped_key` still clean up and the +tests can read the spend rows the proxy wrote for the key. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import tempfile +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal + +from claude_code.cli_driver import run_claude +from client_apps_models import CLAUDE_EVENTS, CODEX_EVENTS, ClaudeEvent, CodexEvent, VercelApi, VercelTurn +from e2e_config import PROXY_BASE_URL, SLOW_PROVIDER_TIMEOUT_SECONDS +from proxy_client import ProxyClient + +SUITE_DIR: Final = Path(__file__).resolve().parent +CLAUDE_CLI: Final = SUITE_DIR / "node_modules" / ".bin" / "claude" +CODEX_CLI: Final = SUITE_DIR / "node_modules" / ".bin" / "codex" +VERCEL_AI_SCRIPT: Final = SUITE_DIR / "vercel_ai_turn.mjs" +NODE: Final = shutil.which("node") or "node" +CODEX_KEY_ENV: Final = "LITELLM_E2E_CODEX_KEY" +PASSTHROUGH_ENV: Final = ("PATH", "TMPDIR", "LANG", "LC_ALL", "LC_CTYPE", "TERM") + + +@dataclass(frozen=True, slots=True) +class CommandFailed: + reason: Literal["nonzero_exit", "timed_out"] + exit_code: int | None + stdout: str + stderr: str + + +def unwrap_run[T](run: T | CommandFailed) -> T: + match run: + case CommandFailed(): + raise AssertionError(run) + case _: + return run + + +def _decoded(chunk: str | bytes | None) -> str: + match chunk: + case bytes(): + return chunk.decode(errors="replace") + case str(): + return chunk + case None: + return "" + + +def _isolated_env(home: Path) -> Mapping[str, str]: + return {**{name: os.environ[name] for name in PASSTHROUGH_ENV if name in os.environ}, "HOME": str(home)} + + +def _run(args: tuple[str, ...], *, env: Mapping[str, str], cwd: Path) -> str | CommandFailed: + try: + completed: Final = subprocess.run( + args, + env=env, + cwd=cwd, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=SLOW_PROVIDER_TIMEOUT_SECONDS, + check=False, + ) + except subprocess.TimeoutExpired as expired: + return CommandFailed( + reason="timed_out", exit_code=None, stdout=_decoded(expired.stdout), stderr=_decoded(expired.stderr) + ) + if completed.returncode != 0: + return CommandFailed( + reason="nonzero_exit", exit_code=completed.returncode, stdout=completed.stdout, stderr=completed.stderr + ) + return completed.stdout + + +def _jsonl_as_array(text: str) -> str: + return "[" + ",".join(line for line in text.splitlines() if line.strip()) + "]" + + +def _codex_config(model: str) -> str: + return "\n".join( + ( + f'model = "{model}"', + 'model_provider = "litellm"', + "", + "[features]", + "plugins = false", + "", + "[model_providers.litellm]", + 'name = "LiteLLM proxy"', + f'base_url = "{PROXY_BASE_URL}/v1"', + f'env_key = "{CODEX_KEY_ENV}"', + 'wire_api = "responses"', + "requires_openai_auth = false", + "", + ) + ) + + +@dataclass(frozen=True, slots=True) +class ClientAppsClient: + proxy: ProxyClient + + def run_claude_code( + self, *, key: str, model: str, prompt: str, allowed_tool: str + ) -> tuple[ClaudeEvent, ...] | CommandFailed: + """One `claude --print --output-format stream-json` turn through + `claude_code/cli_driver.py`, which already isolates HOME and the env. + `--include-partial-messages` surfaces the streamed deltas as events.""" + driven: Final = run_claude( + prompt=prompt, + model=model, + base_url=PROXY_BASE_URL, + api_key=key, + extra_args=("--allowed-tools", allowed_tool, "--permission-mode", "dontAsk", "--include-partial-messages"), + cli_path=str(CLAUDE_CLI), + timeout=SLOW_PROVIDER_TIMEOUT_SECONDS, + ) + if driven.exit_code != 0: + return CommandFailed( + reason="nonzero_exit", exit_code=driven.exit_code, stdout=driven.text, stderr=driven.stderr + ) + return tuple(CLAUDE_EVENTS.validate_python(driven.events)) + + def run_codex(self, *, key: str, model: str, prompt: str) -> tuple[CodexEvent, ...] | CommandFailed: + """One `codex exec --json` turn from a throwaway CODEX_HOME whose + config.toml points the `litellm` provider at the proxy's Responses API.""" + with tempfile.TemporaryDirectory(prefix="e2e-codex-") as home: + codex_home: Final = Path(home) / "codex" + workspace: Final = Path(home) / "workspace" + codex_home.mkdir() + workspace.mkdir() + (codex_home / "config.toml").write_text(_codex_config(model)) + output: Final = _run( + ( + str(CODEX_CLI), + "exec", + "--json", + "--skip-git-repo-check", + "--ephemeral", + "-C", + str(workspace), + prompt, + ), + env={**_isolated_env(Path(home)), "CODEX_HOME": str(codex_home), CODEX_KEY_ENV: key}, + cwd=workspace, + ) + match output: + case CommandFailed(): + return output + case str(): + return tuple(CODEX_EVENTS.validate_json(_jsonl_as_array(output))) + + def run_vercel_ai_sdk( + self, *, key: str, model: str, api: VercelApi, secret_word: str + ) -> VercelTurn | CommandFailed: + """One `streamText` turn of `vercel_ai_turn.mjs` over the chosen API shape, + with a tool whose result is `secret_word`, summarized as JSON on stdout.""" + with tempfile.TemporaryDirectory(prefix="e2e-vercel-ai-") as home: + output: Final = _run( + (NODE, str(VERCEL_AI_SCRIPT)), + env={ + **_isolated_env(Path(home)), + "LITELLM_PROXY_URL": PROXY_BASE_URL, + "LITELLM_API_KEY": key, + "E2E_MODEL": model, + "E2E_API": api, + "E2E_SECRET_WORD": secret_word, + }, + cwd=SUITE_DIR, + ) + match output: + case CommandFailed(): + return output + case str(): + return VercelTurn.model_validate_json(output) diff --git a/tests/e2e/client_apps/client_apps_models.py b/tests/e2e/client_apps/client_apps_models.py new file mode 100644 index 00000000000..bdbce870fc1 --- /dev/null +++ b/tests/e2e/client_apps/client_apps_models.py @@ -0,0 +1,172 @@ +"""What each client app prints, typed down to the fields the tests read: Claude +Code's stream-json events, Codex's `exec --json` events, the summary +`vercel_ai_turn.mjs` writes, and the request body the proxy stores on a spend row.""" + +from __future__ import annotations + +from typing import Annotated, Final, Literal + +from pydantic import BaseModel, Field, JsonValue, TypeAdapter + + +class ToolUseBlock(BaseModel): + type: Literal["tool_use"] + name: str + input: JsonValue = None + + +class ToolResultBlock(BaseModel): + type: Literal["tool_result"] + content: JsonValue = None + is_error: bool = False + + +class OtherBlock(BaseModel): + type: str + + +type ContentBlock = Annotated[ToolUseBlock | ToolResultBlock | OtherBlock, Field(union_mode="left_to_right")] + + +class ClaudeMessage(BaseModel): + content: list[ContentBlock] | str + + +class ClaudeAssistantEvent(BaseModel): + type: Literal["assistant"] + message: ClaudeMessage + + +class ClaudeUserEvent(BaseModel): + type: Literal["user"] + message: ClaudeMessage + + +class StreamDelta(BaseModel): + type: str + + +class ContentBlockDelta(BaseModel): + type: Literal["content_block_delta"] + delta: StreamDelta + + +class OtherStreamPayload(BaseModel): + type: str + + +class ClaudeStreamEvent(BaseModel): + type: Literal["stream_event"] + event: Annotated[ContentBlockDelta | OtherStreamPayload, Field(union_mode="left_to_right")] + + +class ClaudeResultEvent(BaseModel): + type: Literal["result"] + subtype: str + is_error: bool + result: str | None = None + num_turns: int | None = None + total_cost_usd: float | None = None + + +class ClaudeOtherEvent(BaseModel): + type: str + + +type ClaudeEvent = Annotated[ + ClaudeAssistantEvent | ClaudeUserEvent | ClaudeStreamEvent | ClaudeResultEvent | ClaudeOtherEvent, + Field(union_mode="left_to_right"), +] +CLAUDE_EVENTS: Final = TypeAdapter(list[ClaudeEvent]) + + +class CodexCommandExecution(BaseModel): + type: Literal["command_execution"] + command: str + aggregated_output: str + exit_code: int | None = None + status: str + + +class CodexAgentMessage(BaseModel): + type: Literal["agent_message"] + text: str + + +class CodexOtherItem(BaseModel): + type: str + + +type CodexItem = Annotated[ + CodexCommandExecution | CodexAgentMessage | CodexOtherItem, Field(union_mode="left_to_right") +] + + +class CodexItemCompleted(BaseModel): + type: Literal["item.completed"] + item: CodexItem + + +class CodexUsage(BaseModel): + input_tokens: int + output_tokens: int + + +class CodexTurnCompleted(BaseModel): + type: Literal["turn.completed"] + usage: CodexUsage + + +class CodexTurnFailed(BaseModel): + type: Literal["turn.failed"] + error: JsonValue = None + + +class CodexOtherEvent(BaseModel): + type: str + + +type CodexEvent = Annotated[ + CodexItemCompleted | CodexTurnCompleted | CodexTurnFailed | CodexOtherEvent, + Field(union_mode="left_to_right"), +] +CODEX_EVENTS: Final = TypeAdapter(list[CodexEvent]) + + +type VercelApi = Literal["responses", "chat"] + + +class VercelToolCall(BaseModel): + tool_name: str + input: JsonValue = None + + +class SecretWord(BaseModel): + word: str + + +class VercelToolResult(BaseModel): + tool_name: str + output: SecretWord + + +class VercelTurn(BaseModel): + api: VercelApi + model: str + text_deltas: int + tool_calls: list[VercelToolCall] + tool_results: list[VercelToolResult] + errors: list[str] + text: str + steps: int + finish_reason: str + + +class RecordedRequest(BaseModel): + """The client's request body as the proxy stored it on the spend row + (`store_prompts_in_spend_logs: true`): `input` for a Responses call, + `messages` for a chat or Anthropic Messages call.""" + + stream: bool = False + input: JsonValue = None + messages: JsonValue = None diff --git a/tests/e2e/client_apps/conftest.py b/tests/e2e/client_apps/conftest.py new file mode 100644 index 00000000000..d35e8d7a597 --- /dev/null +++ b/tests/e2e/client_apps/conftest.py @@ -0,0 +1,18 @@ +"""`client_apps` suite's `client` fixture. + +Lifecycle (resources/scoped_key), proxy liveness gate, and the e2e/covers +markers all live in the parent tests/e2e/conftest.py. ClientAppsClient holds the +shared ProxyClient so the keys these tests mint tear down through it. +""" + +from __future__ import annotations + +import pytest + +from client_apps_client import ClientAppsClient +from proxy_client import ProxyClient + + +@pytest.fixture(scope="session") +def client(proxy: ProxyClient) -> ClientAppsClient: + return ClientAppsClient(proxy=proxy) diff --git a/tests/e2e/client_apps/package-lock.json b/tests/e2e/client_apps/package-lock.json new file mode 100644 index 00000000000..68751b5e286 --- /dev/null +++ b/tests/e2e/client_apps/package-lock.json @@ -0,0 +1,416 @@ +{ + "name": "litellm-e2e-client-apps", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "litellm-e2e-client-apps", + "dependencies": { + "@ai-sdk/openai": "4.0.72", + "@anthropic-ai/claude-code": "2.1.278", + "@openai/codex": "0.155.1", + "ai": "7.0.109", + "zod": "4.6.5" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@ai-sdk/gateway": { + "version": "4.0.88", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-4.0.88.tgz", + "integrity": "sha512-hpEdOAqrp3FOShSytk6n8BZVZ/xrCV7L9hqYA6Q55h7sQknvh99QfC0uQAIbQWjh+66tmo1Fo6ndQla8uCJGyA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.17", + "@ai-sdk/provider-utils": "5.0.45", + "@vercel/oidc": "3.2.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/openai": { + "version": "4.0.72", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-4.0.72.tgz", + "integrity": "sha512-X5dowqxAXPhu72FpspKX7P5nDycvl/Nt0Qi9A5VjdotV1SrJdhH4vOeWVdBootwEC6uY2hYN0SHAuL1fd4cV8Q==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.17", + "@ai-sdk/provider-utils": "5.0.45" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.17.tgz", + "integrity": "sha512-VYMBxIQdcHqbIf1j+YZlI9Ati6LZ4wJe0GGd4z4a5H/KxTggjeOiyaVYTnfF7LHZK5jMQ+rofmzz4QPqf++NUw==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "5.0.45", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-5.0.45.tgz", + "integrity": "sha512-gLuaCups8OCIRz3eO6WWz5op7VLYNNtFzyH0sWUr0FE9v7cTxR0qxDpQdrKnGcql+9WWttobjnZsJVsuEbsPQQ==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.17", + "@standard-schema/spec": "^1.1.0", + "@workflow/serde": "4.1.0", + "eventsource-parser": "^3.0.8", + "undici": "^7.29.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@anthropic-ai/claude-code": { + "version": "2.1.278", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code/-/claude-code-2.1.278.tgz", + "integrity": "sha512-mfNRqC0GaEXqmP97NiwJBeYBmRuqe2VzgLUreUUaEhyJxJWx2Z6ClW1tBOncGNNXdhj6EY4LPvUIWP+oq311CA==", + "hasInstallScript": true, + "license": "SEE LICENSE IN README.md", + "bin": { + "claude": "bin/claude.exe" + }, + "engines": { + "node": ">=22.0.0" + }, + "optionalDependencies": { + "@anthropic-ai/claude-code-darwin-arm64": "2.1.278", + "@anthropic-ai/claude-code-darwin-x64": "2.1.278", + "@anthropic-ai/claude-code-linux-arm64": "2.1.278", + "@anthropic-ai/claude-code-linux-arm64-musl": "2.1.278", + "@anthropic-ai/claude-code-linux-x64": "2.1.278", + "@anthropic-ai/claude-code-linux-x64-musl": "2.1.278", + "@anthropic-ai/claude-code-win32-arm64": "2.1.278", + "@anthropic-ai/claude-code-win32-x64": "2.1.278" + } + }, + "node_modules/@anthropic-ai/claude-code-darwin-arm64": { + "version": "2.1.278", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-darwin-arm64/-/claude-code-darwin-arm64-2.1.278.tgz", + "integrity": "sha512-Jgl//CpT1KR1N8uxAN0CmkFu3eESU7GYzPOwdGH1pCENysJqYgo2LwsIrTol+EequNMrTG3kkrNW9mIoS8eWLg==", + "cpu": [ + "arm64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@anthropic-ai/claude-code-darwin-x64": { + "version": "2.1.278", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-darwin-x64/-/claude-code-darwin-x64-2.1.278.tgz", + "integrity": "sha512-8ohVNWkdN3wE0P4yFnM7cm7Xd20kbXAHxu+rvqeBNeN8rKyZyPDUx06Zl/7/i9pfR8q9lZMY7kKA3dRxNNxvvg==", + "cpu": [ + "x64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@anthropic-ai/claude-code-linux-arm64": { + "version": "2.1.278", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-linux-arm64/-/claude-code-linux-arm64-2.1.278.tgz", + "integrity": "sha512-QAsHwRVX03nXgAA7pTdvZSLguegbAAPuLG1vY09YHhkukqD6FlkSDwYqK9XFh4up8rFjK7qrzXtokuNULP0nEg==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@anthropic-ai/claude-code-linux-arm64-musl": { + "version": "2.1.278", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-linux-arm64-musl/-/claude-code-linux-arm64-musl-2.1.278.tgz", + "integrity": "sha512-fsroa4Z+4KCQACEYTVRYtnDbJYp3NLLVi53AUHRKKVYNhei92+rQdBxvGB81+ET4BUgfOlkwRt3+fnh7ndP9zg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@anthropic-ai/claude-code-linux-x64": { + "version": "2.1.278", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-linux-x64/-/claude-code-linux-x64-2.1.278.tgz", + "integrity": "sha512-q3r+5aLGAet1MGMkCH2xPsuIW9A40ws4zftURxhYwDenheCKvXc7Gr1jBwvEhYG8uQwgY3YsfNwnvIsh1Bjmeg==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@anthropic-ai/claude-code-linux-x64-musl": { + "version": "2.1.278", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-linux-x64-musl/-/claude-code-linux-x64-musl-2.1.278.tgz", + "integrity": "sha512-fUYSGjtRFWBsxObXd75PaEl+hUsxHdZsWuDEh4WYgyFs9ljiNuPE/PgHh9y/lrysnH0K8HQmMs3mpFvw34KyzA==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@anthropic-ai/claude-code-win32-arm64": { + "version": "2.1.278", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-win32-arm64/-/claude-code-win32-arm64-2.1.278.tgz", + "integrity": "sha512-SQf1ZDCenMPc0730RaZsIR8n15x0kdYiwGsL5QBSIcFjxRpT5gnonf5rZ55aeaO9g2g/ZU2CT99RgqMxy5y0Hg==", + "cpu": [ + "arm64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@anthropic-ai/claude-code-win32-x64": { + "version": "2.1.278", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-win32-x64/-/claude-code-win32-x64-2.1.278.tgz", + "integrity": "sha512-8gwotnfAqgtuayi0m/Ajy5k+fochciv9Erdn7iqaCHM321cz5nOnJy0yCCkk4DsT0nG7kq67dOPXE9N4jdPz7Q==", + "cpu": [ + "x64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@openai/codex": { + "version": "0.155.1", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1.tgz", + "integrity": "sha512-02fAAGyBtlA1zPjEo3kTj/bOSYbPz5DvjLwRZJdV7weFFEDzNFOMjQGmZ/+5CuirYV0hE+AZTrnjzwXYU4AdAQ==", + "license": "Apache-2.0", + "bin": { + "codex": "bin/codex.js" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@openai/codex-darwin-arm64": "npm:@openai/codex@0.155.1-darwin-arm64", + "@openai/codex-darwin-x64": "npm:@openai/codex@0.155.1-darwin-x64", + "@openai/codex-linux-arm64": "npm:@openai/codex@0.155.1-linux-arm64", + "@openai/codex-linux-x64": "npm:@openai/codex@0.155.1-linux-x64", + "@openai/codex-win32-arm64": "npm:@openai/codex@0.155.1-win32-arm64", + "@openai/codex-win32-x64": "npm:@openai/codex@0.155.1-win32-x64" + } + }, + "node_modules/@openai/codex-darwin-arm64": { + "name": "@openai/codex", + "version": "0.155.1-darwin-arm64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-darwin-arm64.tgz", + "integrity": "sha512-cYxzGcRRoBrncyHlR8ed4yXwcoVJZC1pipGULSyJkGFKXJw/Uu57BklvzayuAptjJIipamnOk32CfUkk1F0bLw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@openai/codex-darwin-x64": { + "name": "@openai/codex", + "version": "0.155.1-darwin-x64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-darwin-x64.tgz", + "integrity": "sha512-FDpc+PdELYlyDnhd76Ckm6jNLF+1n3x34Ygd4QLQger810Vkxx/InQ5LY5jwkecJYKcbvyhMmuxTspaj1dLZrA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@openai/codex-linux-arm64": { + "name": "@openai/codex", + "version": "0.155.1-linux-arm64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-linux-arm64.tgz", + "integrity": "sha512-X3fRXm2orhJ3KeB8LgKym4XDUiQaqaOGuaa181bcHTsQI7C8m6tcQQbQsKDzT/2IibikzgYL82jvsgMbq43jww==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@openai/codex-linux-x64": { + "name": "@openai/codex", + "version": "0.155.1-linux-x64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-linux-x64.tgz", + "integrity": "sha512-atv3HF0mubqB0J/XkQ2JopqKzXJ+/7aQtTB2MkJ9MrraujMIz8zbCCLylLkN3PzpVGTJzzQFN/wD1oq8oJPJKg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@openai/codex-win32-arm64": { + "name": "@openai/codex", + "version": "0.155.1-win32-arm64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-win32-arm64.tgz", + "integrity": "sha512-k5x8VO1aF8Xx/nuh1P31TeBgs11WA6i2GJiHqx5YCndFbWzOzUz6aeBaq9+PO6Qfe/Ivennh3I1FKJBU6Q8mpg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@openai/codex-win32-x64": { + "name": "@openai/codex", + "version": "0.155.1-win32-x64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-win32-x64.tgz", + "integrity": "sha512-MO+cCZrgU0Ec7lJP/5NsTe5obJ9/qtRMkQUK0jYWTY1omxLA3lp5IOD2IAmsejlEJB931XRo51LZ7hl178CDjA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/@vercel/oidc": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz", + "integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, + "node_modules/@workflow/serde": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@workflow/serde/-/serde-4.1.0.tgz", + "integrity": "sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==", + "license": "Apache-2.0" + }, + "node_modules/ai": { + "version": "7.0.109", + "resolved": "https://registry.npmjs.org/ai/-/ai-7.0.109.tgz", + "integrity": "sha512-hTnm8K4GIT0R6gHrnDJHr1KRnGVAtFa26mLtjqn/TGy+ufD5Y5XXerTEabLXVgmb9iunWxSFRCPOZUZ/VeXYLQ==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "4.0.88", + "@ai-sdk/provider": "4.0.17", + "@ai-sdk/provider-utils": "5.0.45" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/undici": { + "version": "7.29.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz", + "integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/tests/e2e/client_apps/package.json b/tests/e2e/client_apps/package.json new file mode 100644 index 00000000000..365de0f459c --- /dev/null +++ b/tests/e2e/client_apps/package.json @@ -0,0 +1,15 @@ +{ + "name": "litellm-e2e-client-apps", + "private": true, + "type": "module", + "engines": { + "node": ">=22" + }, + "dependencies": { + "@ai-sdk/openai": "4.0.72", + "@anthropic-ai/claude-code": "2.1.278", + "@openai/codex": "0.155.1", + "ai": "7.0.109", + "zod": "4.6.5" + } +} diff --git a/tests/e2e/client_apps/spend_rows.py b/tests/e2e/client_apps/spend_rows.py new file mode 100644 index 00000000000..282b4250d57 --- /dev/null +++ b/tests/e2e/client_apps/spend_rows.py @@ -0,0 +1,21 @@ +"""The proxy-side half of every client_apps cell: the spend rows the proxy wrote +for the key, each a success with real spend and a request body the client sent +with `stream: true`. Every client here makes at least two calls per tool turn +(the tool call, then the reply), so the poll waits for two rows.""" + +from __future__ import annotations + +from client_apps_models import RecordedRequest +from proxy_client import ProxyClient, SpendLogRow + + +def recorded_request(row: SpendLogRow) -> RecordedRequest: + return RecordedRequest.model_validate(row.proxy_server_request) + + +def streamed_tool_turn_rows(proxy: ProxyClient, key: str) -> list[SpendLogRow]: + rows = proxy.poll_logs_for_key(key, min_rows=2) + assert all(row.status == "success" for row in rows), rows + assert all(row.spend is not None and row.spend > 0 for row in rows), rows + assert all(recorded_request(row).stream for row in rows), rows + return rows diff --git a/tests/e2e/client_apps/test_claude_code_cli_e2e.py b/tests/e2e/client_apps/test_claude_code_cli_e2e.py new file mode 100644 index 00000000000..b3de2ea2a09 --- /dev/null +++ b/tests/e2e/client_apps/test_claude_code_cli_e2e.py @@ -0,0 +1,74 @@ +"""The Claude Code CLI, pinned in this suite's package.json, runs one headless +turn against the proxy on a virtual key: it streams over `/v1/messages`, calls +the Bash tool, reads the tool result, and answers. Client side the stream-json +events carry the tool_use, the tool_result, the streamed deltas, and the final +result; proxy side the spend rows carry real spend and `stream: true`.""" + +from __future__ import annotations + +import json +from typing import Final + +import pytest + +from client_apps_client import ClientAppsClient, unwrap_run +from client_apps_models import ( + ClaudeAssistantEvent, + ClaudeResultEvent, + ClaudeStreamEvent, + ClaudeUserEvent, + ContentBlockDelta, + ToolResultBlock, + ToolUseBlock, +) +from e2e_config import CHEAP_ANTHROPIC_MODEL +from spend_rows import streamed_tool_turn_rows + +pytestmark = pytest.mark.e2e + +PROMPT: Final = "Use the Bash tool to run the command `echo pong` and report what it printed." + + +@pytest.mark.covers("other.client_apps.claude_code_cli.tool_turn_streams") +def test_claude_code_cli_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str) -> None: + events = unwrap_run( + client.run_claude_code( + key=scoped_key, model=CHEAP_ANTHROPIC_MODEL, prompt=PROMPT, allowed_tool="Bash(echo pong)" + ) + ) + + tool_uses = [ + block + for event in events + if isinstance(event, ClaudeAssistantEvent) and isinstance(event.message.content, list) + for block in event.message.content + if isinstance(block, ToolUseBlock) + ] + assert [block.name for block in tool_uses] == ["Bash"], events + + tool_results = [ + block + for event in events + if isinstance(event, ClaudeUserEvent) and isinstance(event.message.content, list) + for block in event.message.content + if isinstance(block, ToolResultBlock) + ] + assert len(tool_results) == 1, events + assert not tool_results[0].is_error, tool_results[0] + assert "pong" in json.dumps(tool_results[0].content), tool_results[0] + + delta_types = [ + event.event.delta.type + for event in events + if isinstance(event, ClaudeStreamEvent) and isinstance(event.event, ContentBlockDelta) + ] + assert "input_json_delta" in delta_types, delta_types + assert "text_delta" in delta_types, delta_types + + results = [event for event in events if isinstance(event, ClaudeResultEvent)] + assert len(results) == 1, events + assert results[0].subtype == "success", results[0] + assert not results[0].is_error, results[0] + assert results[0].result is not None and "pong" in results[0].result, results[0] + + streamed_tool_turn_rows(client.proxy, scoped_key) diff --git a/tests/e2e/client_apps/test_codex_cli_e2e.py b/tests/e2e/client_apps/test_codex_cli_e2e.py new file mode 100644 index 00000000000..3f0ea644149 --- /dev/null +++ b/tests/e2e/client_apps/test_codex_cli_e2e.py @@ -0,0 +1,43 @@ +"""The Codex CLI, pinned in this suite's package.json, runs one headless +`codex exec --json` turn against the proxy on a virtual key through a custom +model provider (`wire_api = "responses"`, so `/v1/responses`): it runs a shell +command, reads its output, and answers. Client side the JSONL events carry the +completed command execution and the agent message; proxy side the spend rows +carry real spend and `stream: true` (Codex always streams the Responses API).""" + +from __future__ import annotations + +from typing import Final + +import pytest + +from client_apps_client import ClientAppsClient, unwrap_run +from client_apps_models import CodexAgentMessage, CodexCommandExecution, CodexItemCompleted, CodexTurnCompleted +from e2e_config import CHEAP_OPENAI_MODEL +from spend_rows import streamed_tool_turn_rows + +pytestmark = pytest.mark.e2e + +PROMPT: Final = "Run the shell command `echo pong` and reply with exactly what it printed." + + +@pytest.mark.covers("other.client_apps.codex_cli.tool_turn_streams") +def test_codex_cli_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str) -> None: + events = unwrap_run(client.run_codex(key=scoped_key, model=CHEAP_OPENAI_MODEL, prompt=PROMPT)) + + completed_items = [event.item for event in events if isinstance(event, CodexItemCompleted)] + commands = [item for item in completed_items if isinstance(item, CodexCommandExecution)] + assert len(commands) == 1, events + assert commands[0].status == "completed", commands[0] + assert commands[0].exit_code == 0, commands[0] + assert "pong" in commands[0].aggregated_output, commands[0] + + messages = [item for item in completed_items if isinstance(item, CodexAgentMessage)] + assert len(messages) == 1, events + assert "pong" in messages[0].text, messages[0] + + turns = [event for event in events if isinstance(event, CodexTurnCompleted)] + assert len(turns) == 1, events + assert turns[0].usage.output_tokens > 0, turns[0] + + streamed_tool_turn_rows(client.proxy, scoped_key) diff --git a/tests/e2e/client_apps/test_vercel_ai_sdk_e2e.py b/tests/e2e/client_apps/test_vercel_ai_sdk_e2e.py new file mode 100644 index 00000000000..06ae33fbf81 --- /dev/null +++ b/tests/e2e/client_apps/test_vercel_ai_sdk_e2e.py @@ -0,0 +1,52 @@ +"""The Vercel AI SDK (`ai` + `@ai-sdk/openai`, pinned in this suite's +package.json) runs one `streamText` turn against the proxy on a virtual key, +once over `/v1/responses` and once over `/v1/chat/completions`: the model calls +a local tool whose result is a secret word only that tool knows, then answers +with the word. Client side the summary carries the streamed text deltas, the +tool call, the tool result, and the final text; proxy side the spend rows carry +real spend, `stream: true`, and the request body shape of the API used.""" + +from __future__ import annotations + +import pytest + +from client_apps_client import ClientAppsClient, unwrap_run +from client_apps_models import VercelApi +from e2e_config import CHEAP_OPENAI_MODEL, unique_marker +from spend_rows import recorded_request, streamed_tool_turn_rows + +pytestmark = pytest.mark.e2e + + +@pytest.mark.parametrize( + "api", + [ + pytest.param( + "responses", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_responses.tool_turn_streams") + ), + pytest.param( + "chat", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_chat_completions.tool_turn_streams") + ), + ], +) +def test_vercel_ai_sdk_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str, api: VercelApi) -> None: + secret_word = f"pong-{unique_marker()}" + turn = unwrap_run( + client.run_vercel_ai_sdk(key=scoped_key, model=CHEAP_OPENAI_MODEL, api=api, secret_word=secret_word) + ) + + assert turn.errors == [], turn + assert [call.tool_name for call in turn.tool_calls] == ["reveal_secret_word"], turn + assert [result.output.word for result in turn.tool_results] == [secret_word], turn + assert secret_word in turn.text, turn + assert turn.text_deltas >= 2, turn + assert turn.steps == 2, turn + assert turn.finish_reason == "stop", turn + + rows = streamed_tool_turn_rows(client.proxy, scoped_key) + bodies = [recorded_request(row) for row in rows] + match api: + case "responses": + assert all(body.input is not None for body in bodies), rows + case "chat": + assert all(body.messages is not None for body in bodies), rows diff --git a/tests/e2e/client_apps/vercel_ai_turn.mjs b/tests/e2e/client_apps/vercel_ai_turn.mjs new file mode 100644 index 00000000000..7b8e27058b6 --- /dev/null +++ b/tests/e2e/client_apps/vercel_ai_turn.mjs @@ -0,0 +1,78 @@ +import { createOpenAI } from "@ai-sdk/openai"; +import { stepCountIs, streamText, tool } from "ai"; +import { z } from "zod"; + +const required = (name) => { + const value = process.env[name]; + if (!value) { + throw new Error(`${name} must be set`); + } + return value; +}; + +const proxyUrl = required("LITELLM_PROXY_URL"); +const apiKey = required("LITELLM_API_KEY"); +const modelId = required("E2E_MODEL"); +const api = required("E2E_API"); +const secretWord = required("E2E_SECRET_WORD"); + +const openai = createOpenAI({ baseURL: `${proxyUrl}/v1`, apiKey }); +const modelByApi = { + responses: () => openai.responses(modelId), + chat: () => openai.chat(modelId), +}; +if (!(api in modelByApi)) { + throw new Error(`E2E_API must be one of ${Object.keys(modelByApi).join(", ")}, got ${api}`); +} + +const result = streamText({ + model: modelByApi[api](), + prompt: + "Call the reveal_secret_word tool once, then reply with exactly the word it returns and nothing else.", + tools: { + reveal_secret_word: tool({ + description: "Reveals the secret word the user is waiting for.", + inputSchema: z.object({ + reason: z.string().describe("Why the word is needed"), + }), + execute: async () => ({ word: secretWord }), + }), + }, + stopWhen: stepCountIs(2), +}); + +const summary = { + api, + model: modelId, + text_deltas: 0, + tool_calls: [], + tool_results: [], + errors: [], +}; +for await (const part of result.fullStream) { + switch (part.type) { + case "text-delta": + summary.text_deltas += 1; + break; + case "tool-call": + summary.tool_calls.push({ tool_name: part.toolName, input: part.input }); + break; + case "tool-result": + summary.tool_results.push({ tool_name: part.toolName, output: part.output }); + break; + case "error": + summary.errors.push(String(part.error?.message ?? part.error)); + break; + default: + break; + } +} + +process.stdout.write( + JSON.stringify({ + ...summary, + text: await result.text, + steps: (await result.steps).length, + finish_reason: await result.finishReason, + }), +); diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index 9292e5f07db..7580cdd24f3 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -54,3 +54,8 @@ - {id: other.auth.jwt.wrong_issuer_denied, module: other, tier: P0, area: auth, assertions: [wrong_issuer_denied], source: "auth/handle_jwt.py", rationale: "A signed token with the correct audience and an unexpected issuer is rejected"} - {id: other.auth.jwt.wrong_audience_denied, module: other, tier: P0, area: auth, assertions: [wrong_audience_denied], source: "auth/handle_jwt.py", rationale: "A signed token from the trusted issuer intended for another app is rejected"} + +- {id: other.client_apps.claude_code_cli.tool_turn_streams, module: other, tier: P1, area: client_apps, assertions: [tool_turn_streams], source: "anthropic_endpoints/endpoints.py /v1/messages", rationale: "The Claude Code CLI on a virtual key completes one streamed turn with a Bash tool call, and the proxy bills every streamed call"} +- {id: other.client_apps.codex_cli.tool_turn_streams, module: other, tier: P1, area: client_apps, assertions: [tool_turn_streams], source: "response_api_endpoints/endpoints.py /v1/responses", rationale: "The Codex CLI on a virtual key completes one streamed turn with a shell command, and the proxy bills every streamed call"} +- {id: other.client_apps.vercel_ai_sdk_responses.tool_turn_streams, module: other, tier: P1, area: client_apps, assertions: [tool_turn_streams], source: "response_api_endpoints/endpoints.py /v1/responses", rationale: "A Vercel AI SDK streamText turn over the Responses API on a virtual key calls a tool and answers with its result, and the proxy bills every streamed call"} +- {id: other.client_apps.vercel_ai_sdk_chat_completions.tool_turn_streams, module: other, tier: P1, area: client_apps, assertions: [tool_turn_streams], source: "proxy_server.py /v1/chat/completions", rationale: "A Vercel AI SDK streamText turn over chat completions on a virtual key calls a tool and answers with its result, and the proxy bills every streamed call"}