test(e2e): drive the proxy through the Claude Code CLI, the Codex CLI, and the Vercel AI SDK

This commit is contained in:
mateo-berri 2026-09-21 22:01:11 -07:00
parent 19c13ac971
commit 173f4c45ed
14 changed files with 1096 additions and 0 deletions

View file

@ -129,6 +129,17 @@ jobs:
- name: Install Playwright chromium
run: uv run --no-sync playwright install --with-deps chromium
- name: Set up Node for the client_apps suite
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version-file: tests/e2e/client_apps/.nvmrc
cache: npm
cache-dependency-path: tests/e2e/client_apps/package-lock.json
- name: Install the client apps the client_apps suite drives
working-directory: tests/e2e/client_apps
run: npm ci --no-audit --no-fund
- name: Configure AWS credentials
id: aws
uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0

View file

@ -48,6 +48,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
- `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite
- `gateway/` - proxy configuration only (`litellm-config.yml`); no tests
- `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke
- `client_apps/` - real client applications driven headlessly against the proxy on a virtual key, one streamed turn with a tool call each: the Claude Code CLI (`/v1/messages`), the Codex CLI (`/v1/responses`), and a Vercel AI SDK script (`vercel_ai_turn.mjs`, over both `/v1/responses` and `/v1/chat/completions`). The CLIs and the SDK are pinned npm dependencies of the suite's own `package.json` (`npm ci` in this folder before running; `test-e2e-changed.yml` does it in CI), the Claude Code path reuses `claude_code/cli_driver.py`, and every `.py` file here follows the Python rules in this file
- `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`
## MCP suite: real Datadog only

View file

@ -0,0 +1 @@
24.19.0

View file

@ -0,0 +1,189 @@
"""Client for the `client_apps` suite: runs the Claude Code CLI and the Codex CLI
installed into this folder by `npm ci`, and the Vercel AI SDK script
`vercel_ai_turn.mjs`, headlessly against the proxy on a virtual key. Each run gets
an empty HOME and only an allowlisted environment, reads nothing from stdin, and
comes back as the typed events or summary it printed, or a `CommandFailed` value.
Holds the shared ProxyClient so `resources` / `scoped_key` still clean up and the
tests can read the spend rows the proxy wrote for the key.
"""
from __future__ import annotations
import os
import shutil
import subprocess
import tempfile
from collections.abc import Mapping
from dataclasses import dataclass
from pathlib import Path
from typing import Final, Literal
from claude_code.cli_driver import run_claude
from client_apps_models import CLAUDE_EVENTS, CODEX_EVENTS, ClaudeEvent, CodexEvent, VercelApi, VercelTurn
from e2e_config import PROXY_BASE_URL, SLOW_PROVIDER_TIMEOUT_SECONDS
from proxy_client import ProxyClient
SUITE_DIR: Final = Path(__file__).resolve().parent
CLAUDE_CLI: Final = SUITE_DIR / "node_modules" / ".bin" / "claude"
CODEX_CLI: Final = SUITE_DIR / "node_modules" / ".bin" / "codex"
VERCEL_AI_SCRIPT: Final = SUITE_DIR / "vercel_ai_turn.mjs"
NODE: Final = shutil.which("node") or "node"
CODEX_KEY_ENV: Final = "LITELLM_E2E_CODEX_KEY"
PASSTHROUGH_ENV: Final = ("PATH", "TMPDIR", "LANG", "LC_ALL", "LC_CTYPE", "TERM")
@dataclass(frozen=True, slots=True)
class CommandFailed:
reason: Literal["nonzero_exit", "timed_out"]
exit_code: int | None
stdout: str
stderr: str
def unwrap_run[T](run: T | CommandFailed) -> T:
match run:
case CommandFailed():
raise AssertionError(run)
case _:
return run
def _decoded(chunk: str | bytes | None) -> str:
match chunk:
case bytes():
return chunk.decode(errors="replace")
case str():
return chunk
case None:
return ""
def _isolated_env(home: Path) -> Mapping[str, str]:
return {**{name: os.environ[name] for name in PASSTHROUGH_ENV if name in os.environ}, "HOME": str(home)}
def _run(args: tuple[str, ...], *, env: Mapping[str, str], cwd: Path) -> str | CommandFailed:
try:
completed: Final = subprocess.run(
args,
env=env,
cwd=cwd,
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=SLOW_PROVIDER_TIMEOUT_SECONDS,
check=False,
)
except subprocess.TimeoutExpired as expired:
return CommandFailed(
reason="timed_out", exit_code=None, stdout=_decoded(expired.stdout), stderr=_decoded(expired.stderr)
)
if completed.returncode != 0:
return CommandFailed(
reason="nonzero_exit", exit_code=completed.returncode, stdout=completed.stdout, stderr=completed.stderr
)
return completed.stdout
def _jsonl_as_array(text: str) -> str:
return "[" + ",".join(line for line in text.splitlines() if line.strip()) + "]"
def _codex_config(model: str) -> str:
return "\n".join(
(
f'model = "{model}"',
'model_provider = "litellm"',
"",
"[features]",
"plugins = false",
"",
"[model_providers.litellm]",
'name = "LiteLLM proxy"',
f'base_url = "{PROXY_BASE_URL}/v1"',
f'env_key = "{CODEX_KEY_ENV}"',
'wire_api = "responses"',
"requires_openai_auth = false",
"",
)
)
@dataclass(frozen=True, slots=True)
class ClientAppsClient:
proxy: ProxyClient
def run_claude_code(
self, *, key: str, model: str, prompt: str, allowed_tool: str
) -> tuple[ClaudeEvent, ...] | CommandFailed:
"""One `claude --print --output-format stream-json` turn through
`claude_code/cli_driver.py`, which already isolates HOME and the env.
`--include-partial-messages` surfaces the streamed deltas as events."""
driven: Final = run_claude(
prompt=prompt,
model=model,
base_url=PROXY_BASE_URL,
api_key=key,
extra_args=("--allowed-tools", allowed_tool, "--permission-mode", "dontAsk", "--include-partial-messages"),
cli_path=str(CLAUDE_CLI),
timeout=SLOW_PROVIDER_TIMEOUT_SECONDS,
)
if driven.exit_code != 0:
return CommandFailed(
reason="nonzero_exit", exit_code=driven.exit_code, stdout=driven.text, stderr=driven.stderr
)
return tuple(CLAUDE_EVENTS.validate_python(driven.events))
def run_codex(self, *, key: str, model: str, prompt: str) -> tuple[CodexEvent, ...] | CommandFailed:
"""One `codex exec --json` turn from a throwaway CODEX_HOME whose
config.toml points the `litellm` provider at the proxy's Responses API."""
with tempfile.TemporaryDirectory(prefix="e2e-codex-") as home:
codex_home: Final = Path(home) / "codex"
workspace: Final = Path(home) / "workspace"
codex_home.mkdir()
workspace.mkdir()
(codex_home / "config.toml").write_text(_codex_config(model))
output: Final = _run(
(
str(CODEX_CLI),
"exec",
"--json",
"--skip-git-repo-check",
"--ephemeral",
"-C",
str(workspace),
prompt,
),
env={**_isolated_env(Path(home)), "CODEX_HOME": str(codex_home), CODEX_KEY_ENV: key},
cwd=workspace,
)
match output:
case CommandFailed():
return output
case str():
return tuple(CODEX_EVENTS.validate_json(_jsonl_as_array(output)))
def run_vercel_ai_sdk(
self, *, key: str, model: str, api: VercelApi, secret_word: str
) -> VercelTurn | CommandFailed:
"""One `streamText` turn of `vercel_ai_turn.mjs` over the chosen API shape,
with a tool whose result is `secret_word`, summarized as JSON on stdout."""
with tempfile.TemporaryDirectory(prefix="e2e-vercel-ai-") as home:
output: Final = _run(
(NODE, str(VERCEL_AI_SCRIPT)),
env={
**_isolated_env(Path(home)),
"LITELLM_PROXY_URL": PROXY_BASE_URL,
"LITELLM_API_KEY": key,
"E2E_MODEL": model,
"E2E_API": api,
"E2E_SECRET_WORD": secret_word,
},
cwd=SUITE_DIR,
)
match output:
case CommandFailed():
return output
case str():
return VercelTurn.model_validate_json(output)

View file

@ -0,0 +1,172 @@
"""What each client app prints, typed down to the fields the tests read: Claude
Code's stream-json events, Codex's `exec --json` events, the summary
`vercel_ai_turn.mjs` writes, and the request body the proxy stores on a spend row."""
from __future__ import annotations
from typing import Annotated, Final, Literal
from pydantic import BaseModel, Field, JsonValue, TypeAdapter
class ToolUseBlock(BaseModel):
type: Literal["tool_use"]
name: str
input: JsonValue = None
class ToolResultBlock(BaseModel):
type: Literal["tool_result"]
content: JsonValue = None
is_error: bool = False
class OtherBlock(BaseModel):
type: str
type ContentBlock = Annotated[ToolUseBlock | ToolResultBlock | OtherBlock, Field(union_mode="left_to_right")]
class ClaudeMessage(BaseModel):
content: list[ContentBlock] | str
class ClaudeAssistantEvent(BaseModel):
type: Literal["assistant"]
message: ClaudeMessage
class ClaudeUserEvent(BaseModel):
type: Literal["user"]
message: ClaudeMessage
class StreamDelta(BaseModel):
type: str
class ContentBlockDelta(BaseModel):
type: Literal["content_block_delta"]
delta: StreamDelta
class OtherStreamPayload(BaseModel):
type: str
class ClaudeStreamEvent(BaseModel):
type: Literal["stream_event"]
event: Annotated[ContentBlockDelta | OtherStreamPayload, Field(union_mode="left_to_right")]
class ClaudeResultEvent(BaseModel):
type: Literal["result"]
subtype: str
is_error: bool
result: str | None = None
num_turns: int | None = None
total_cost_usd: float | None = None
class ClaudeOtherEvent(BaseModel):
type: str
type ClaudeEvent = Annotated[
ClaudeAssistantEvent | ClaudeUserEvent | ClaudeStreamEvent | ClaudeResultEvent | ClaudeOtherEvent,
Field(union_mode="left_to_right"),
]
CLAUDE_EVENTS: Final = TypeAdapter(list[ClaudeEvent])
class CodexCommandExecution(BaseModel):
type: Literal["command_execution"]
command: str
aggregated_output: str
exit_code: int | None = None
status: str
class CodexAgentMessage(BaseModel):
type: Literal["agent_message"]
text: str
class CodexOtherItem(BaseModel):
type: str
type CodexItem = Annotated[
CodexCommandExecution | CodexAgentMessage | CodexOtherItem, Field(union_mode="left_to_right")
]
class CodexItemCompleted(BaseModel):
type: Literal["item.completed"]
item: CodexItem
class CodexUsage(BaseModel):
input_tokens: int
output_tokens: int
class CodexTurnCompleted(BaseModel):
type: Literal["turn.completed"]
usage: CodexUsage
class CodexTurnFailed(BaseModel):
type: Literal["turn.failed"]
error: JsonValue = None
class CodexOtherEvent(BaseModel):
type: str
type CodexEvent = Annotated[
CodexItemCompleted | CodexTurnCompleted | CodexTurnFailed | CodexOtherEvent,
Field(union_mode="left_to_right"),
]
CODEX_EVENTS: Final = TypeAdapter(list[CodexEvent])
type VercelApi = Literal["responses", "chat"]
class VercelToolCall(BaseModel):
tool_name: str
input: JsonValue = None
class SecretWord(BaseModel):
word: str
class VercelToolResult(BaseModel):
tool_name: str
output: SecretWord
class VercelTurn(BaseModel):
api: VercelApi
model: str
text_deltas: int
tool_calls: list[VercelToolCall]
tool_results: list[VercelToolResult]
errors: list[str]
text: str
steps: int
finish_reason: str
class RecordedRequest(BaseModel):
"""The client's request body as the proxy stored it on the spend row
(`store_prompts_in_spend_logs: true`): `input` for a Responses call,
`messages` for a chat or Anthropic Messages call."""
stream: bool = False
input: JsonValue = None
messages: JsonValue = None

View file

@ -0,0 +1,18 @@
"""`client_apps` suite's `client` fixture.
Lifecycle (resources/scoped_key), proxy liveness gate, and the e2e/covers
markers all live in the parent tests/e2e/conftest.py. ClientAppsClient holds the
shared ProxyClient so the keys these tests mint tear down through it.
"""
from __future__ import annotations
import pytest
from client_apps_client import ClientAppsClient
from proxy_client import ProxyClient
@pytest.fixture(scope="session")
def client(proxy: ProxyClient) -> ClientAppsClient:
return ClientAppsClient(proxy=proxy)

416
tests/e2e/client_apps/package-lock.json generated Normal file
View file

@ -0,0 +1,416 @@
{
"name": "litellm-e2e-client-apps",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "litellm-e2e-client-apps",
"dependencies": {
"@ai-sdk/openai": "4.0.72",
"@anthropic-ai/claude-code": "2.1.278",
"@openai/codex": "0.155.1",
"ai": "7.0.109",
"zod": "4.6.5"
},
"engines": {
"node": ">=22"
}
},
"node_modules/@ai-sdk/gateway": {
"version": "4.0.88",
"resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-4.0.88.tgz",
"integrity": "sha512-hpEdOAqrp3FOShSytk6n8BZVZ/xrCV7L9hqYA6Q55h7sQknvh99QfC0uQAIbQWjh+66tmo1Fo6ndQla8uCJGyA==",
"license": "Apache-2.0",
"dependencies": {
"@ai-sdk/provider": "4.0.17",
"@ai-sdk/provider-utils": "5.0.45",
"@vercel/oidc": "3.2.0"
},
"engines": {
"node": ">=22"
},
"peerDependencies": {
"zod": "^3.25.76 || ^4.1.8"
}
},
"node_modules/@ai-sdk/openai": {
"version": "4.0.72",
"resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-4.0.72.tgz",
"integrity": "sha512-X5dowqxAXPhu72FpspKX7P5nDycvl/Nt0Qi9A5VjdotV1SrJdhH4vOeWVdBootwEC6uY2hYN0SHAuL1fd4cV8Q==",
"license": "Apache-2.0",
"dependencies": {
"@ai-sdk/provider": "4.0.17",
"@ai-sdk/provider-utils": "5.0.45"
},
"engines": {
"node": ">=22"
},
"peerDependencies": {
"zod": "^3.25.76 || ^4.1.8"
}
},
"node_modules/@ai-sdk/provider": {
"version": "4.0.17",
"resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.17.tgz",
"integrity": "sha512-VYMBxIQdcHqbIf1j+YZlI9Ati6LZ4wJe0GGd4z4a5H/KxTggjeOiyaVYTnfF7LHZK5jMQ+rofmzz4QPqf++NUw==",
"license": "Apache-2.0",
"dependencies": {
"json-schema": "^0.4.0"
},
"engines": {
"node": ">=22"
}
},
"node_modules/@ai-sdk/provider-utils": {
"version": "5.0.45",
"resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-5.0.45.tgz",
"integrity": "sha512-gLuaCups8OCIRz3eO6WWz5op7VLYNNtFzyH0sWUr0FE9v7cTxR0qxDpQdrKnGcql+9WWttobjnZsJVsuEbsPQQ==",
"license": "Apache-2.0",
"dependencies": {
"@ai-sdk/provider": "4.0.17",
"@standard-schema/spec": "^1.1.0",
"@workflow/serde": "4.1.0",
"eventsource-parser": "^3.0.8",
"undici": "^7.29.0"
},
"engines": {
"node": ">=22"
},
"peerDependencies": {
"zod": "^3.25.76 || ^4.1.8"
}
},
"node_modules/@anthropic-ai/claude-code": {
"version": "2.1.278",
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code/-/claude-code-2.1.278.tgz",
"integrity": "sha512-mfNRqC0GaEXqmP97NiwJBeYBmRuqe2VzgLUreUUaEhyJxJWx2Z6ClW1tBOncGNNXdhj6EY4LPvUIWP+oq311CA==",
"hasInstallScript": true,
"license": "SEE LICENSE IN README.md",
"bin": {
"claude": "bin/claude.exe"
},
"engines": {
"node": ">=22.0.0"
},
"optionalDependencies": {
"@anthropic-ai/claude-code-darwin-arm64": "2.1.278",
"@anthropic-ai/claude-code-darwin-x64": "2.1.278",
"@anthropic-ai/claude-code-linux-arm64": "2.1.278",
"@anthropic-ai/claude-code-linux-arm64-musl": "2.1.278",
"@anthropic-ai/claude-code-linux-x64": "2.1.278",
"@anthropic-ai/claude-code-linux-x64-musl": "2.1.278",
"@anthropic-ai/claude-code-win32-arm64": "2.1.278",
"@anthropic-ai/claude-code-win32-x64": "2.1.278"
}
},
"node_modules/@anthropic-ai/claude-code-darwin-arm64": {
"version": "2.1.278",
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-darwin-arm64/-/claude-code-darwin-arm64-2.1.278.tgz",
"integrity": "sha512-Jgl//CpT1KR1N8uxAN0CmkFu3eESU7GYzPOwdGH1pCENysJqYgo2LwsIrTol+EequNMrTG3kkrNW9mIoS8eWLg==",
"cpu": [
"arm64"
],
"license": "SEE LICENSE IN LICENSE.md",
"optional": true,
"os": [
"darwin"
]
},
"node_modules/@anthropic-ai/claude-code-darwin-x64": {
"version": "2.1.278",
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-darwin-x64/-/claude-code-darwin-x64-2.1.278.tgz",
"integrity": "sha512-8ohVNWkdN3wE0P4yFnM7cm7Xd20kbXAHxu+rvqeBNeN8rKyZyPDUx06Zl/7/i9pfR8q9lZMY7kKA3dRxNNxvvg==",
"cpu": [
"x64"
],
"license": "SEE LICENSE IN LICENSE.md",
"optional": true,
"os": [
"darwin"
]
},
"node_modules/@anthropic-ai/claude-code-linux-arm64": {
"version": "2.1.278",
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-linux-arm64/-/claude-code-linux-arm64-2.1.278.tgz",
"integrity": "sha512-QAsHwRVX03nXgAA7pTdvZSLguegbAAPuLG1vY09YHhkukqD6FlkSDwYqK9XFh4up8rFjK7qrzXtokuNULP0nEg==",
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "SEE LICENSE IN LICENSE.md",
"optional": true,
"os": [
"linux"
]
},
"node_modules/@anthropic-ai/claude-code-linux-arm64-musl": {
"version": "2.1.278",
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-linux-arm64-musl/-/claude-code-linux-arm64-musl-2.1.278.tgz",
"integrity": "sha512-fsroa4Z+4KCQACEYTVRYtnDbJYp3NLLVi53AUHRKKVYNhei92+rQdBxvGB81+ET4BUgfOlkwRt3+fnh7ndP9zg==",
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "SEE LICENSE IN LICENSE.md",
"optional": true,
"os": [
"linux"
]
},
"node_modules/@anthropic-ai/claude-code-linux-x64": {
"version": "2.1.278",
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-linux-x64/-/claude-code-linux-x64-2.1.278.tgz",
"integrity": "sha512-q3r+5aLGAet1MGMkCH2xPsuIW9A40ws4zftURxhYwDenheCKvXc7Gr1jBwvEhYG8uQwgY3YsfNwnvIsh1Bjmeg==",
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "SEE LICENSE IN LICENSE.md",
"optional": true,
"os": [
"linux"
]
},
"node_modules/@anthropic-ai/claude-code-linux-x64-musl": {
"version": "2.1.278",
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-linux-x64-musl/-/claude-code-linux-x64-musl-2.1.278.tgz",
"integrity": "sha512-fUYSGjtRFWBsxObXd75PaEl+hUsxHdZsWuDEh4WYgyFs9ljiNuPE/PgHh9y/lrysnH0K8HQmMs3mpFvw34KyzA==",
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "SEE LICENSE IN LICENSE.md",
"optional": true,
"os": [
"linux"
]
},
"node_modules/@anthropic-ai/claude-code-win32-arm64": {
"version": "2.1.278",
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-win32-arm64/-/claude-code-win32-arm64-2.1.278.tgz",
"integrity": "sha512-SQf1ZDCenMPc0730RaZsIR8n15x0kdYiwGsL5QBSIcFjxRpT5gnonf5rZ55aeaO9g2g/ZU2CT99RgqMxy5y0Hg==",
"cpu": [
"arm64"
],
"license": "SEE LICENSE IN LICENSE.md",
"optional": true,
"os": [
"win32"
]
},
"node_modules/@anthropic-ai/claude-code-win32-x64": {
"version": "2.1.278",
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-win32-x64/-/claude-code-win32-x64-2.1.278.tgz",
"integrity": "sha512-8gwotnfAqgtuayi0m/Ajy5k+fochciv9Erdn7iqaCHM321cz5nOnJy0yCCkk4DsT0nG7kq67dOPXE9N4jdPz7Q==",
"cpu": [
"x64"
],
"license": "SEE LICENSE IN LICENSE.md",
"optional": true,
"os": [
"win32"
]
},
"node_modules/@openai/codex": {
"version": "0.155.1",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1.tgz",
"integrity": "sha512-02fAAGyBtlA1zPjEo3kTj/bOSYbPz5DvjLwRZJdV7weFFEDzNFOMjQGmZ/+5CuirYV0hE+AZTrnjzwXYU4AdAQ==",
"license": "Apache-2.0",
"bin": {
"codex": "bin/codex.js"
},
"engines": {
"node": ">=16"
},
"optionalDependencies": {
"@openai/codex-darwin-arm64": "npm:@openai/codex@0.155.1-darwin-arm64",
"@openai/codex-darwin-x64": "npm:@openai/codex@0.155.1-darwin-x64",
"@openai/codex-linux-arm64": "npm:@openai/codex@0.155.1-linux-arm64",
"@openai/codex-linux-x64": "npm:@openai/codex@0.155.1-linux-x64",
"@openai/codex-win32-arm64": "npm:@openai/codex@0.155.1-win32-arm64",
"@openai/codex-win32-x64": "npm:@openai/codex@0.155.1-win32-x64"
}
},
"node_modules/@openai/codex-darwin-arm64": {
"name": "@openai/codex",
"version": "0.155.1-darwin-arm64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-darwin-arm64.tgz",
"integrity": "sha512-cYxzGcRRoBrncyHlR8ed4yXwcoVJZC1pipGULSyJkGFKXJw/Uu57BklvzayuAptjJIipamnOk32CfUkk1F0bLw==",
"cpu": [
"arm64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=16"
}
},
"node_modules/@openai/codex-darwin-x64": {
"name": "@openai/codex",
"version": "0.155.1-darwin-x64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-darwin-x64.tgz",
"integrity": "sha512-FDpc+PdELYlyDnhd76Ckm6jNLF+1n3x34Ygd4QLQger810Vkxx/InQ5LY5jwkecJYKcbvyhMmuxTspaj1dLZrA==",
"cpu": [
"x64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=16"
}
},
"node_modules/@openai/codex-linux-arm64": {
"name": "@openai/codex",
"version": "0.155.1-linux-arm64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-linux-arm64.tgz",
"integrity": "sha512-X3fRXm2orhJ3KeB8LgKym4XDUiQaqaOGuaa181bcHTsQI7C8m6tcQQbQsKDzT/2IibikzgYL82jvsgMbq43jww==",
"cpu": [
"arm64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=16"
}
},
"node_modules/@openai/codex-linux-x64": {
"name": "@openai/codex",
"version": "0.155.1-linux-x64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-linux-x64.tgz",
"integrity": "sha512-atv3HF0mubqB0J/XkQ2JopqKzXJ+/7aQtTB2MkJ9MrraujMIz8zbCCLylLkN3PzpVGTJzzQFN/wD1oq8oJPJKg==",
"cpu": [
"x64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=16"
}
},
"node_modules/@openai/codex-win32-arm64": {
"name": "@openai/codex",
"version": "0.155.1-win32-arm64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-win32-arm64.tgz",
"integrity": "sha512-k5x8VO1aF8Xx/nuh1P31TeBgs11WA6i2GJiHqx5YCndFbWzOzUz6aeBaq9+PO6Qfe/Ivennh3I1FKJBU6Q8mpg==",
"cpu": [
"arm64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=16"
}
},
"node_modules/@openai/codex-win32-x64": {
"name": "@openai/codex",
"version": "0.155.1-win32-x64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-win32-x64.tgz",
"integrity": "sha512-MO+cCZrgU0Ec7lJP/5NsTe5obJ9/qtRMkQUK0jYWTY1omxLA3lp5IOD2IAmsejlEJB931XRo51LZ7hl178CDjA==",
"cpu": [
"x64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=16"
}
},
"node_modules/@standard-schema/spec": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz",
"integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==",
"license": "MIT"
},
"node_modules/@vercel/oidc": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz",
"integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==",
"license": "Apache-2.0",
"engines": {
"node": ">= 20"
}
},
"node_modules/@workflow/serde": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/@workflow/serde/-/serde-4.1.0.tgz",
"integrity": "sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==",
"license": "Apache-2.0"
},
"node_modules/ai": {
"version": "7.0.109",
"resolved": "https://registry.npmjs.org/ai/-/ai-7.0.109.tgz",
"integrity": "sha512-hTnm8K4GIT0R6gHrnDJHr1KRnGVAtFa26mLtjqn/TGy+ufD5Y5XXerTEabLXVgmb9iunWxSFRCPOZUZ/VeXYLQ==",
"license": "Apache-2.0",
"dependencies": {
"@ai-sdk/gateway": "4.0.88",
"@ai-sdk/provider": "4.0.17",
"@ai-sdk/provider-utils": "5.0.45"
},
"engines": {
"node": ">=22"
},
"peerDependencies": {
"zod": "^3.25.76 || ^4.1.8"
}
},
"node_modules/eventsource-parser": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz",
"integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==",
"license": "MIT",
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/json-schema": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz",
"integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==",
"license": "(AFL-2.1 OR BSD-3-Clause)"
},
"node_modules/undici": {
"version": "7.29.1",
"resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz",
"integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==",
"license": "MIT",
"engines": {
"node": ">=20.18.1"
}
},
"node_modules/zod": {
"version": "4.6.5",
"resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz",
"integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==",
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/colinhacks"
}
}
}
}

View file

@ -0,0 +1,15 @@
{
"name": "litellm-e2e-client-apps",
"private": true,
"type": "module",
"engines": {
"node": ">=22"
},
"dependencies": {
"@ai-sdk/openai": "4.0.72",
"@anthropic-ai/claude-code": "2.1.278",
"@openai/codex": "0.155.1",
"ai": "7.0.109",
"zod": "4.6.5"
}
}

View file

@ -0,0 +1,21 @@
"""The proxy-side half of every client_apps cell: the spend rows the proxy wrote
for the key, each a success with real spend and a request body the client sent
with `stream: true`. Every client here makes at least two calls per tool turn
(the tool call, then the reply), so the poll waits for two rows."""
from __future__ import annotations
from client_apps_models import RecordedRequest
from proxy_client import ProxyClient, SpendLogRow
def recorded_request(row: SpendLogRow) -> RecordedRequest:
return RecordedRequest.model_validate(row.proxy_server_request)
def streamed_tool_turn_rows(proxy: ProxyClient, key: str) -> list[SpendLogRow]:
rows = proxy.poll_logs_for_key(key, min_rows=2)
assert all(row.status == "success" for row in rows), rows
assert all(row.spend is not None and row.spend > 0 for row in rows), rows
assert all(recorded_request(row).stream for row in rows), rows
return rows

View file

@ -0,0 +1,74 @@
"""The Claude Code CLI, pinned in this suite's package.json, runs one headless
turn against the proxy on a virtual key: it streams over `/v1/messages`, calls
the Bash tool, reads the tool result, and answers. Client side the stream-json
events carry the tool_use, the tool_result, the streamed deltas, and the final
result; proxy side the spend rows carry real spend and `stream: true`."""
from __future__ import annotations
import json
from typing import Final
import pytest
from client_apps_client import ClientAppsClient, unwrap_run
from client_apps_models import (
ClaudeAssistantEvent,
ClaudeResultEvent,
ClaudeStreamEvent,
ClaudeUserEvent,
ContentBlockDelta,
ToolResultBlock,
ToolUseBlock,
)
from e2e_config import CHEAP_ANTHROPIC_MODEL
from spend_rows import streamed_tool_turn_rows
pytestmark = pytest.mark.e2e
PROMPT: Final = "Use the Bash tool to run the command `echo pong` and report what it printed."
@pytest.mark.covers("other.client_apps.claude_code_cli.tool_turn_streams")
def test_claude_code_cli_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str) -> None:
events = unwrap_run(
client.run_claude_code(
key=scoped_key, model=CHEAP_ANTHROPIC_MODEL, prompt=PROMPT, allowed_tool="Bash(echo pong)"
)
)
tool_uses = [
block
for event in events
if isinstance(event, ClaudeAssistantEvent) and isinstance(event.message.content, list)
for block in event.message.content
if isinstance(block, ToolUseBlock)
]
assert [block.name for block in tool_uses] == ["Bash"], events
tool_results = [
block
for event in events
if isinstance(event, ClaudeUserEvent) and isinstance(event.message.content, list)
for block in event.message.content
if isinstance(block, ToolResultBlock)
]
assert len(tool_results) == 1, events
assert not tool_results[0].is_error, tool_results[0]
assert "pong" in json.dumps(tool_results[0].content), tool_results[0]
delta_types = [
event.event.delta.type
for event in events
if isinstance(event, ClaudeStreamEvent) and isinstance(event.event, ContentBlockDelta)
]
assert "input_json_delta" in delta_types, delta_types
assert "text_delta" in delta_types, delta_types
results = [event for event in events if isinstance(event, ClaudeResultEvent)]
assert len(results) == 1, events
assert results[0].subtype == "success", results[0]
assert not results[0].is_error, results[0]
assert results[0].result is not None and "pong" in results[0].result, results[0]
streamed_tool_turn_rows(client.proxy, scoped_key)

View file

@ -0,0 +1,43 @@
"""The Codex CLI, pinned in this suite's package.json, runs one headless
`codex exec --json` turn against the proxy on a virtual key through a custom
model provider (`wire_api = "responses"`, so `/v1/responses`): it runs a shell
command, reads its output, and answers. Client side the JSONL events carry the
completed command execution and the agent message; proxy side the spend rows
carry real spend and `stream: true` (Codex always streams the Responses API)."""
from __future__ import annotations
from typing import Final
import pytest
from client_apps_client import ClientAppsClient, unwrap_run
from client_apps_models import CodexAgentMessage, CodexCommandExecution, CodexItemCompleted, CodexTurnCompleted
from e2e_config import CHEAP_OPENAI_MODEL
from spend_rows import streamed_tool_turn_rows
pytestmark = pytest.mark.e2e
PROMPT: Final = "Run the shell command `echo pong` and reply with exactly what it printed."
@pytest.mark.covers("other.client_apps.codex_cli.tool_turn_streams")
def test_codex_cli_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str) -> None:
events = unwrap_run(client.run_codex(key=scoped_key, model=CHEAP_OPENAI_MODEL, prompt=PROMPT))
completed_items = [event.item for event in events if isinstance(event, CodexItemCompleted)]
commands = [item for item in completed_items if isinstance(item, CodexCommandExecution)]
assert len(commands) == 1, events
assert commands[0].status == "completed", commands[0]
assert commands[0].exit_code == 0, commands[0]
assert "pong" in commands[0].aggregated_output, commands[0]
messages = [item for item in completed_items if isinstance(item, CodexAgentMessage)]
assert len(messages) == 1, events
assert "pong" in messages[0].text, messages[0]
turns = [event for event in events if isinstance(event, CodexTurnCompleted)]
assert len(turns) == 1, events
assert turns[0].usage.output_tokens > 0, turns[0]
streamed_tool_turn_rows(client.proxy, scoped_key)

View file

@ -0,0 +1,52 @@
"""The Vercel AI SDK (`ai` + `@ai-sdk/openai`, pinned in this suite's
package.json) runs one `streamText` turn against the proxy on a virtual key,
once over `/v1/responses` and once over `/v1/chat/completions`: the model calls
a local tool whose result is a secret word only that tool knows, then answers
with the word. Client side the summary carries the streamed text deltas, the
tool call, the tool result, and the final text; proxy side the spend rows carry
real spend, `stream: true`, and the request body shape of the API used."""
from __future__ import annotations
import pytest
from client_apps_client import ClientAppsClient, unwrap_run
from client_apps_models import VercelApi
from e2e_config import CHEAP_OPENAI_MODEL, unique_marker
from spend_rows import recorded_request, streamed_tool_turn_rows
pytestmark = pytest.mark.e2e
@pytest.mark.parametrize(
"api",
[
pytest.param(
"responses", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_responses.tool_turn_streams")
),
pytest.param(
"chat", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_chat_completions.tool_turn_streams")
),
],
)
def test_vercel_ai_sdk_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str, api: VercelApi) -> None:
secret_word = f"pong-{unique_marker()}"
turn = unwrap_run(
client.run_vercel_ai_sdk(key=scoped_key, model=CHEAP_OPENAI_MODEL, api=api, secret_word=secret_word)
)
assert turn.errors == [], turn
assert [call.tool_name for call in turn.tool_calls] == ["reveal_secret_word"], turn
assert [result.output.word for result in turn.tool_results] == [secret_word], turn
assert secret_word in turn.text, turn
assert turn.text_deltas >= 2, turn
assert turn.steps == 2, turn
assert turn.finish_reason == "stop", turn
rows = streamed_tool_turn_rows(client.proxy, scoped_key)
bodies = [recorded_request(row) for row in rows]
match api:
case "responses":
assert all(body.input is not None for body in bodies), rows
case "chat":
assert all(body.messages is not None for body in bodies), rows

View file

@ -0,0 +1,78 @@
import { createOpenAI } from "@ai-sdk/openai";
import { stepCountIs, streamText, tool } from "ai";
import { z } from "zod";
const required = (name) => {
const value = process.env[name];
if (!value) {
throw new Error(`${name} must be set`);
}
return value;
};
const proxyUrl = required("LITELLM_PROXY_URL");
const apiKey = required("LITELLM_API_KEY");
const modelId = required("E2E_MODEL");
const api = required("E2E_API");
const secretWord = required("E2E_SECRET_WORD");
const openai = createOpenAI({ baseURL: `${proxyUrl}/v1`, apiKey });
const modelByApi = {
responses: () => openai.responses(modelId),
chat: () => openai.chat(modelId),
};
if (!(api in modelByApi)) {
throw new Error(`E2E_API must be one of ${Object.keys(modelByApi).join(", ")}, got ${api}`);
}
const result = streamText({
model: modelByApi[api](),
prompt:
"Call the reveal_secret_word tool once, then reply with exactly the word it returns and nothing else.",
tools: {
reveal_secret_word: tool({
description: "Reveals the secret word the user is waiting for.",
inputSchema: z.object({
reason: z.string().describe("Why the word is needed"),
}),
execute: async () => ({ word: secretWord }),
}),
},
stopWhen: stepCountIs(2),
});
const summary = {
api,
model: modelId,
text_deltas: 0,
tool_calls: [],
tool_results: [],
errors: [],
};
for await (const part of result.fullStream) {
switch (part.type) {
case "text-delta":
summary.text_deltas += 1;
break;
case "tool-call":
summary.tool_calls.push({ tool_name: part.toolName, input: part.input });
break;
case "tool-result":
summary.tool_results.push({ tool_name: part.toolName, output: part.output });
break;
case "error":
summary.errors.push(String(part.error?.message ?? part.error));
break;
default:
break;
}
}
process.stdout.write(
JSON.stringify({
...summary,
text: await result.text,
steps: (await result.steps).length,
finish_reason: await result.finishReason,
}),
);

View file

@ -54,3 +54,8 @@
- {id: other.auth.jwt.wrong_issuer_denied, module: other, tier: P0, area: auth, assertions: [wrong_issuer_denied], source: "auth/handle_jwt.py", rationale: "A signed token with the correct audience and an unexpected issuer is rejected"}
- {id: other.auth.jwt.wrong_audience_denied, module: other, tier: P0, area: auth, assertions: [wrong_audience_denied], source: "auth/handle_jwt.py", rationale: "A signed token from the trusted issuer intended for another app is rejected"}
- {id: other.client_apps.claude_code_cli.tool_turn_streams, module: other, tier: P1, area: client_apps, assertions: [tool_turn_streams], source: "anthropic_endpoints/endpoints.py /v1/messages", rationale: "The Claude Code CLI on a virtual key completes one streamed turn with a Bash tool call, and the proxy bills every streamed call"}
- {id: other.client_apps.codex_cli.tool_turn_streams, module: other, tier: P1, area: client_apps, assertions: [tool_turn_streams], source: "response_api_endpoints/endpoints.py /v1/responses", rationale: "The Codex CLI on a virtual key completes one streamed turn with a shell command, and the proxy bills every streamed call"}
- {id: other.client_apps.vercel_ai_sdk_responses.tool_turn_streams, module: other, tier: P1, area: client_apps, assertions: [tool_turn_streams], source: "response_api_endpoints/endpoints.py /v1/responses", rationale: "A Vercel AI SDK streamText turn over the Responses API on a virtual key calls a tool and answers with its result, and the proxy bills every streamed call"}
- {id: other.client_apps.vercel_ai_sdk_chat_completions.tool_turn_streams, module: other, tier: P1, area: client_apps, assertions: [tool_turn_streams], source: "proxy_server.py /v1/chat/completions", rationale: "A Vercel AI SDK streamText turn over chat completions on a virtual key calls a tool and answers with its result, and the proxy bills every streamed call"}