mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
test(e2e): drive the proxy through the Claude Code CLI, the Codex CLI, and the Vercel AI SDK
This commit is contained in:
parent
19c13ac971
commit
173f4c45ed
14 changed files with 1096 additions and 0 deletions
11
.github/workflows/test-e2e-changed.yml
vendored
11
.github/workflows/test-e2e-changed.yml
vendored
|
|
@ -129,6 +129,17 @@ jobs:
|
|||
- name: Install Playwright chromium
|
||||
run: uv run --no-sync playwright install --with-deps chromium
|
||||
|
||||
- name: Set up Node for the client_apps suite
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
|
||||
with:
|
||||
node-version-file: tests/e2e/client_apps/.nvmrc
|
||||
cache: npm
|
||||
cache-dependency-path: tests/e2e/client_apps/package-lock.json
|
||||
|
||||
- name: Install the client apps the client_apps suite drives
|
||||
working-directory: tests/e2e/client_apps
|
||||
run: npm ci --no-audit --no-fund
|
||||
|
||||
- name: Configure AWS credentials
|
||||
id: aws
|
||||
uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
|
|||
- `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite
|
||||
- `gateway/` - proxy configuration only (`litellm-config.yml`); no tests
|
||||
- `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke
|
||||
- `client_apps/` - real client applications driven headlessly against the proxy on a virtual key, one streamed turn with a tool call each: the Claude Code CLI (`/v1/messages`), the Codex CLI (`/v1/responses`), and a Vercel AI SDK script (`vercel_ai_turn.mjs`, over both `/v1/responses` and `/v1/chat/completions`). The CLIs and the SDK are pinned npm dependencies of the suite's own `package.json` (`npm ci` in this folder before running; `test-e2e-changed.yml` does it in CI), the Claude Code path reuses `claude_code/cli_driver.py`, and every `.py` file here follows the Python rules in this file
|
||||
- `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`
|
||||
|
||||
## MCP suite: real Datadog only
|
||||
|
|
|
|||
1
tests/e2e/client_apps/.nvmrc
Normal file
1
tests/e2e/client_apps/.nvmrc
Normal file
|
|
@ -0,0 +1 @@
|
|||
24.19.0
|
||||
189
tests/e2e/client_apps/client_apps_client.py
Normal file
189
tests/e2e/client_apps/client_apps_client.py
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
"""Client for the `client_apps` suite: runs the Claude Code CLI and the Codex CLI
|
||||
installed into this folder by `npm ci`, and the Vercel AI SDK script
|
||||
`vercel_ai_turn.mjs`, headlessly against the proxy on a virtual key. Each run gets
|
||||
an empty HOME and only an allowlisted environment, reads nothing from stdin, and
|
||||
comes back as the typed events or summary it printed, or a `CommandFailed` value.
|
||||
|
||||
Holds the shared ProxyClient so `resources` / `scoped_key` still clean up and the
|
||||
tests can read the spend rows the proxy wrote for the key.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final, Literal
|
||||
|
||||
from claude_code.cli_driver import run_claude
|
||||
from client_apps_models import CLAUDE_EVENTS, CODEX_EVENTS, ClaudeEvent, CodexEvent, VercelApi, VercelTurn
|
||||
from e2e_config import PROXY_BASE_URL, SLOW_PROVIDER_TIMEOUT_SECONDS
|
||||
from proxy_client import ProxyClient
|
||||
|
||||
SUITE_DIR: Final = Path(__file__).resolve().parent
|
||||
CLAUDE_CLI: Final = SUITE_DIR / "node_modules" / ".bin" / "claude"
|
||||
CODEX_CLI: Final = SUITE_DIR / "node_modules" / ".bin" / "codex"
|
||||
VERCEL_AI_SCRIPT: Final = SUITE_DIR / "vercel_ai_turn.mjs"
|
||||
NODE: Final = shutil.which("node") or "node"
|
||||
CODEX_KEY_ENV: Final = "LITELLM_E2E_CODEX_KEY"
|
||||
PASSTHROUGH_ENV: Final = ("PATH", "TMPDIR", "LANG", "LC_ALL", "LC_CTYPE", "TERM")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CommandFailed:
|
||||
reason: Literal["nonzero_exit", "timed_out"]
|
||||
exit_code: int | None
|
||||
stdout: str
|
||||
stderr: str
|
||||
|
||||
|
||||
def unwrap_run[T](run: T | CommandFailed) -> T:
|
||||
match run:
|
||||
case CommandFailed():
|
||||
raise AssertionError(run)
|
||||
case _:
|
||||
return run
|
||||
|
||||
|
||||
def _decoded(chunk: str | bytes | None) -> str:
|
||||
match chunk:
|
||||
case bytes():
|
||||
return chunk.decode(errors="replace")
|
||||
case str():
|
||||
return chunk
|
||||
case None:
|
||||
return ""
|
||||
|
||||
|
||||
def _isolated_env(home: Path) -> Mapping[str, str]:
|
||||
return {**{name: os.environ[name] for name in PASSTHROUGH_ENV if name in os.environ}, "HOME": str(home)}
|
||||
|
||||
|
||||
def _run(args: tuple[str, ...], *, env: Mapping[str, str], cwd: Path) -> str | CommandFailed:
|
||||
try:
|
||||
completed: Final = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
cwd=cwd,
|
||||
stdin=subprocess.DEVNULL,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=SLOW_PROVIDER_TIMEOUT_SECONDS,
|
||||
check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired as expired:
|
||||
return CommandFailed(
|
||||
reason="timed_out", exit_code=None, stdout=_decoded(expired.stdout), stderr=_decoded(expired.stderr)
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
return CommandFailed(
|
||||
reason="nonzero_exit", exit_code=completed.returncode, stdout=completed.stdout, stderr=completed.stderr
|
||||
)
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def _jsonl_as_array(text: str) -> str:
|
||||
return "[" + ",".join(line for line in text.splitlines() if line.strip()) + "]"
|
||||
|
||||
|
||||
def _codex_config(model: str) -> str:
|
||||
return "\n".join(
|
||||
(
|
||||
f'model = "{model}"',
|
||||
'model_provider = "litellm"',
|
||||
"",
|
||||
"[features]",
|
||||
"plugins = false",
|
||||
"",
|
||||
"[model_providers.litellm]",
|
||||
'name = "LiteLLM proxy"',
|
||||
f'base_url = "{PROXY_BASE_URL}/v1"',
|
||||
f'env_key = "{CODEX_KEY_ENV}"',
|
||||
'wire_api = "responses"',
|
||||
"requires_openai_auth = false",
|
||||
"",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ClientAppsClient:
|
||||
proxy: ProxyClient
|
||||
|
||||
def run_claude_code(
|
||||
self, *, key: str, model: str, prompt: str, allowed_tool: str
|
||||
) -> tuple[ClaudeEvent, ...] | CommandFailed:
|
||||
"""One `claude --print --output-format stream-json` turn through
|
||||
`claude_code/cli_driver.py`, which already isolates HOME and the env.
|
||||
`--include-partial-messages` surfaces the streamed deltas as events."""
|
||||
driven: Final = run_claude(
|
||||
prompt=prompt,
|
||||
model=model,
|
||||
base_url=PROXY_BASE_URL,
|
||||
api_key=key,
|
||||
extra_args=("--allowed-tools", allowed_tool, "--permission-mode", "dontAsk", "--include-partial-messages"),
|
||||
cli_path=str(CLAUDE_CLI),
|
||||
timeout=SLOW_PROVIDER_TIMEOUT_SECONDS,
|
||||
)
|
||||
if driven.exit_code != 0:
|
||||
return CommandFailed(
|
||||
reason="nonzero_exit", exit_code=driven.exit_code, stdout=driven.text, stderr=driven.stderr
|
||||
)
|
||||
return tuple(CLAUDE_EVENTS.validate_python(driven.events))
|
||||
|
||||
def run_codex(self, *, key: str, model: str, prompt: str) -> tuple[CodexEvent, ...] | CommandFailed:
|
||||
"""One `codex exec --json` turn from a throwaway CODEX_HOME whose
|
||||
config.toml points the `litellm` provider at the proxy's Responses API."""
|
||||
with tempfile.TemporaryDirectory(prefix="e2e-codex-") as home:
|
||||
codex_home: Final = Path(home) / "codex"
|
||||
workspace: Final = Path(home) / "workspace"
|
||||
codex_home.mkdir()
|
||||
workspace.mkdir()
|
||||
(codex_home / "config.toml").write_text(_codex_config(model))
|
||||
output: Final = _run(
|
||||
(
|
||||
str(CODEX_CLI),
|
||||
"exec",
|
||||
"--json",
|
||||
"--skip-git-repo-check",
|
||||
"--ephemeral",
|
||||
"-C",
|
||||
str(workspace),
|
||||
prompt,
|
||||
),
|
||||
env={**_isolated_env(Path(home)), "CODEX_HOME": str(codex_home), CODEX_KEY_ENV: key},
|
||||
cwd=workspace,
|
||||
)
|
||||
match output:
|
||||
case CommandFailed():
|
||||
return output
|
||||
case str():
|
||||
return tuple(CODEX_EVENTS.validate_json(_jsonl_as_array(output)))
|
||||
|
||||
def run_vercel_ai_sdk(
|
||||
self, *, key: str, model: str, api: VercelApi, secret_word: str
|
||||
) -> VercelTurn | CommandFailed:
|
||||
"""One `streamText` turn of `vercel_ai_turn.mjs` over the chosen API shape,
|
||||
with a tool whose result is `secret_word`, summarized as JSON on stdout."""
|
||||
with tempfile.TemporaryDirectory(prefix="e2e-vercel-ai-") as home:
|
||||
output: Final = _run(
|
||||
(NODE, str(VERCEL_AI_SCRIPT)),
|
||||
env={
|
||||
**_isolated_env(Path(home)),
|
||||
"LITELLM_PROXY_URL": PROXY_BASE_URL,
|
||||
"LITELLM_API_KEY": key,
|
||||
"E2E_MODEL": model,
|
||||
"E2E_API": api,
|
||||
"E2E_SECRET_WORD": secret_word,
|
||||
},
|
||||
cwd=SUITE_DIR,
|
||||
)
|
||||
match output:
|
||||
case CommandFailed():
|
||||
return output
|
||||
case str():
|
||||
return VercelTurn.model_validate_json(output)
|
||||
172
tests/e2e/client_apps/client_apps_models.py
Normal file
172
tests/e2e/client_apps/client_apps_models.py
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
"""What each client app prints, typed down to the fields the tests read: Claude
|
||||
Code's stream-json events, Codex's `exec --json` events, the summary
|
||||
`vercel_ai_turn.mjs` writes, and the request body the proxy stores on a spend row."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Final, Literal
|
||||
|
||||
from pydantic import BaseModel, Field, JsonValue, TypeAdapter
|
||||
|
||||
|
||||
class ToolUseBlock(BaseModel):
|
||||
type: Literal["tool_use"]
|
||||
name: str
|
||||
input: JsonValue = None
|
||||
|
||||
|
||||
class ToolResultBlock(BaseModel):
|
||||
type: Literal["tool_result"]
|
||||
content: JsonValue = None
|
||||
is_error: bool = False
|
||||
|
||||
|
||||
class OtherBlock(BaseModel):
|
||||
type: str
|
||||
|
||||
|
||||
type ContentBlock = Annotated[ToolUseBlock | ToolResultBlock | OtherBlock, Field(union_mode="left_to_right")]
|
||||
|
||||
|
||||
class ClaudeMessage(BaseModel):
|
||||
content: list[ContentBlock] | str
|
||||
|
||||
|
||||
class ClaudeAssistantEvent(BaseModel):
|
||||
type: Literal["assistant"]
|
||||
message: ClaudeMessage
|
||||
|
||||
|
||||
class ClaudeUserEvent(BaseModel):
|
||||
type: Literal["user"]
|
||||
message: ClaudeMessage
|
||||
|
||||
|
||||
class StreamDelta(BaseModel):
|
||||
type: str
|
||||
|
||||
|
||||
class ContentBlockDelta(BaseModel):
|
||||
type: Literal["content_block_delta"]
|
||||
delta: StreamDelta
|
||||
|
||||
|
||||
class OtherStreamPayload(BaseModel):
|
||||
type: str
|
||||
|
||||
|
||||
class ClaudeStreamEvent(BaseModel):
|
||||
type: Literal["stream_event"]
|
||||
event: Annotated[ContentBlockDelta | OtherStreamPayload, Field(union_mode="left_to_right")]
|
||||
|
||||
|
||||
class ClaudeResultEvent(BaseModel):
|
||||
type: Literal["result"]
|
||||
subtype: str
|
||||
is_error: bool
|
||||
result: str | None = None
|
||||
num_turns: int | None = None
|
||||
total_cost_usd: float | None = None
|
||||
|
||||
|
||||
class ClaudeOtherEvent(BaseModel):
|
||||
type: str
|
||||
|
||||
|
||||
type ClaudeEvent = Annotated[
|
||||
ClaudeAssistantEvent | ClaudeUserEvent | ClaudeStreamEvent | ClaudeResultEvent | ClaudeOtherEvent,
|
||||
Field(union_mode="left_to_right"),
|
||||
]
|
||||
CLAUDE_EVENTS: Final = TypeAdapter(list[ClaudeEvent])
|
||||
|
||||
|
||||
class CodexCommandExecution(BaseModel):
|
||||
type: Literal["command_execution"]
|
||||
command: str
|
||||
aggregated_output: str
|
||||
exit_code: int | None = None
|
||||
status: str
|
||||
|
||||
|
||||
class CodexAgentMessage(BaseModel):
|
||||
type: Literal["agent_message"]
|
||||
text: str
|
||||
|
||||
|
||||
class CodexOtherItem(BaseModel):
|
||||
type: str
|
||||
|
||||
|
||||
type CodexItem = Annotated[
|
||||
CodexCommandExecution | CodexAgentMessage | CodexOtherItem, Field(union_mode="left_to_right")
|
||||
]
|
||||
|
||||
|
||||
class CodexItemCompleted(BaseModel):
|
||||
type: Literal["item.completed"]
|
||||
item: CodexItem
|
||||
|
||||
|
||||
class CodexUsage(BaseModel):
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
|
||||
|
||||
class CodexTurnCompleted(BaseModel):
|
||||
type: Literal["turn.completed"]
|
||||
usage: CodexUsage
|
||||
|
||||
|
||||
class CodexTurnFailed(BaseModel):
|
||||
type: Literal["turn.failed"]
|
||||
error: JsonValue = None
|
||||
|
||||
|
||||
class CodexOtherEvent(BaseModel):
|
||||
type: str
|
||||
|
||||
|
||||
type CodexEvent = Annotated[
|
||||
CodexItemCompleted | CodexTurnCompleted | CodexTurnFailed | CodexOtherEvent,
|
||||
Field(union_mode="left_to_right"),
|
||||
]
|
||||
CODEX_EVENTS: Final = TypeAdapter(list[CodexEvent])
|
||||
|
||||
|
||||
type VercelApi = Literal["responses", "chat"]
|
||||
|
||||
|
||||
class VercelToolCall(BaseModel):
|
||||
tool_name: str
|
||||
input: JsonValue = None
|
||||
|
||||
|
||||
class SecretWord(BaseModel):
|
||||
word: str
|
||||
|
||||
|
||||
class VercelToolResult(BaseModel):
|
||||
tool_name: str
|
||||
output: SecretWord
|
||||
|
||||
|
||||
class VercelTurn(BaseModel):
|
||||
api: VercelApi
|
||||
model: str
|
||||
text_deltas: int
|
||||
tool_calls: list[VercelToolCall]
|
||||
tool_results: list[VercelToolResult]
|
||||
errors: list[str]
|
||||
text: str
|
||||
steps: int
|
||||
finish_reason: str
|
||||
|
||||
|
||||
class RecordedRequest(BaseModel):
|
||||
"""The client's request body as the proxy stored it on the spend row
|
||||
(`store_prompts_in_spend_logs: true`): `input` for a Responses call,
|
||||
`messages` for a chat or Anthropic Messages call."""
|
||||
|
||||
stream: bool = False
|
||||
input: JsonValue = None
|
||||
messages: JsonValue = None
|
||||
18
tests/e2e/client_apps/conftest.py
Normal file
18
tests/e2e/client_apps/conftest.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
"""`client_apps` suite's `client` fixture.
|
||||
|
||||
Lifecycle (resources/scoped_key), proxy liveness gate, and the e2e/covers
|
||||
markers all live in the parent tests/e2e/conftest.py. ClientAppsClient holds the
|
||||
shared ProxyClient so the keys these tests mint tear down through it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from client_apps_client import ClientAppsClient
|
||||
from proxy_client import ProxyClient
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def client(proxy: ProxyClient) -> ClientAppsClient:
|
||||
return ClientAppsClient(proxy=proxy)
|
||||
416
tests/e2e/client_apps/package-lock.json
generated
Normal file
416
tests/e2e/client_apps/package-lock.json
generated
Normal file
|
|
@ -0,0 +1,416 @@
|
|||
{
|
||||
"name": "litellm-e2e-client-apps",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "litellm-e2e-client-apps",
|
||||
"dependencies": {
|
||||
"@ai-sdk/openai": "4.0.72",
|
||||
"@anthropic-ai/claude-code": "2.1.278",
|
||||
"@openai/codex": "0.155.1",
|
||||
"ai": "7.0.109",
|
||||
"zod": "4.6.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
}
|
||||
},
|
||||
"node_modules/@ai-sdk/gateway": {
|
||||
"version": "4.0.88",
|
||||
"resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-4.0.88.tgz",
|
||||
"integrity": "sha512-hpEdOAqrp3FOShSytk6n8BZVZ/xrCV7L9hqYA6Q55h7sQknvh99QfC0uQAIbQWjh+66tmo1Fo6ndQla8uCJGyA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@ai-sdk/provider": "4.0.17",
|
||||
"@ai-sdk/provider-utils": "5.0.45",
|
||||
"@vercel/oidc": "3.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"zod": "^3.25.76 || ^4.1.8"
|
||||
}
|
||||
},
|
||||
"node_modules/@ai-sdk/openai": {
|
||||
"version": "4.0.72",
|
||||
"resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-4.0.72.tgz",
|
||||
"integrity": "sha512-X5dowqxAXPhu72FpspKX7P5nDycvl/Nt0Qi9A5VjdotV1SrJdhH4vOeWVdBootwEC6uY2hYN0SHAuL1fd4cV8Q==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@ai-sdk/provider": "4.0.17",
|
||||
"@ai-sdk/provider-utils": "5.0.45"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"zod": "^3.25.76 || ^4.1.8"
|
||||
}
|
||||
},
|
||||
"node_modules/@ai-sdk/provider": {
|
||||
"version": "4.0.17",
|
||||
"resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.17.tgz",
|
||||
"integrity": "sha512-VYMBxIQdcHqbIf1j+YZlI9Ati6LZ4wJe0GGd4z4a5H/KxTggjeOiyaVYTnfF7LHZK5jMQ+rofmzz4QPqf++NUw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"json-schema": "^0.4.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
}
|
||||
},
|
||||
"node_modules/@ai-sdk/provider-utils": {
|
||||
"version": "5.0.45",
|
||||
"resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-5.0.45.tgz",
|
||||
"integrity": "sha512-gLuaCups8OCIRz3eO6WWz5op7VLYNNtFzyH0sWUr0FE9v7cTxR0qxDpQdrKnGcql+9WWttobjnZsJVsuEbsPQQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@ai-sdk/provider": "4.0.17",
|
||||
"@standard-schema/spec": "^1.1.0",
|
||||
"@workflow/serde": "4.1.0",
|
||||
"eventsource-parser": "^3.0.8",
|
||||
"undici": "^7.29.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"zod": "^3.25.76 || ^4.1.8"
|
||||
}
|
||||
},
|
||||
"node_modules/@anthropic-ai/claude-code": {
|
||||
"version": "2.1.278",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code/-/claude-code-2.1.278.tgz",
|
||||
"integrity": "sha512-mfNRqC0GaEXqmP97NiwJBeYBmRuqe2VzgLUreUUaEhyJxJWx2Z6ClW1tBOncGNNXdhj6EY4LPvUIWP+oq311CA==",
|
||||
"hasInstallScript": true,
|
||||
"license": "SEE LICENSE IN README.md",
|
||||
"bin": {
|
||||
"claude": "bin/claude.exe"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@anthropic-ai/claude-code-darwin-arm64": "2.1.278",
|
||||
"@anthropic-ai/claude-code-darwin-x64": "2.1.278",
|
||||
"@anthropic-ai/claude-code-linux-arm64": "2.1.278",
|
||||
"@anthropic-ai/claude-code-linux-arm64-musl": "2.1.278",
|
||||
"@anthropic-ai/claude-code-linux-x64": "2.1.278",
|
||||
"@anthropic-ai/claude-code-linux-x64-musl": "2.1.278",
|
||||
"@anthropic-ai/claude-code-win32-arm64": "2.1.278",
|
||||
"@anthropic-ai/claude-code-win32-x64": "2.1.278"
|
||||
}
|
||||
},
|
||||
"node_modules/@anthropic-ai/claude-code-darwin-arm64": {
|
||||
"version": "2.1.278",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-darwin-arm64/-/claude-code-darwin-arm64-2.1.278.tgz",
|
||||
"integrity": "sha512-Jgl//CpT1KR1N8uxAN0CmkFu3eESU7GYzPOwdGH1pCENysJqYgo2LwsIrTol+EequNMrTG3kkrNW9mIoS8eWLg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "SEE LICENSE IN LICENSE.md",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/@anthropic-ai/claude-code-darwin-x64": {
|
||||
"version": "2.1.278",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-darwin-x64/-/claude-code-darwin-x64-2.1.278.tgz",
|
||||
"integrity": "sha512-8ohVNWkdN3wE0P4yFnM7cm7Xd20kbXAHxu+rvqeBNeN8rKyZyPDUx06Zl/7/i9pfR8q9lZMY7kKA3dRxNNxvvg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "SEE LICENSE IN LICENSE.md",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/@anthropic-ai/claude-code-linux-arm64": {
|
||||
"version": "2.1.278",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-linux-arm64/-/claude-code-linux-arm64-2.1.278.tgz",
|
||||
"integrity": "sha512-QAsHwRVX03nXgAA7pTdvZSLguegbAAPuLG1vY09YHhkukqD6FlkSDwYqK9XFh4up8rFjK7qrzXtokuNULP0nEg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "SEE LICENSE IN LICENSE.md",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@anthropic-ai/claude-code-linux-arm64-musl": {
|
||||
"version": "2.1.278",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-linux-arm64-musl/-/claude-code-linux-arm64-musl-2.1.278.tgz",
|
||||
"integrity": "sha512-fsroa4Z+4KCQACEYTVRYtnDbJYp3NLLVi53AUHRKKVYNhei92+rQdBxvGB81+ET4BUgfOlkwRt3+fnh7ndP9zg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"libc": [
|
||||
"musl"
|
||||
],
|
||||
"license": "SEE LICENSE IN LICENSE.md",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@anthropic-ai/claude-code-linux-x64": {
|
||||
"version": "2.1.278",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-linux-x64/-/claude-code-linux-x64-2.1.278.tgz",
|
||||
"integrity": "sha512-q3r+5aLGAet1MGMkCH2xPsuIW9A40ws4zftURxhYwDenheCKvXc7Gr1jBwvEhYG8uQwgY3YsfNwnvIsh1Bjmeg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "SEE LICENSE IN LICENSE.md",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@anthropic-ai/claude-code-linux-x64-musl": {
|
||||
"version": "2.1.278",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-linux-x64-musl/-/claude-code-linux-x64-musl-2.1.278.tgz",
|
||||
"integrity": "sha512-fUYSGjtRFWBsxObXd75PaEl+hUsxHdZsWuDEh4WYgyFs9ljiNuPE/PgHh9y/lrysnH0K8HQmMs3mpFvw34KyzA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"libc": [
|
||||
"musl"
|
||||
],
|
||||
"license": "SEE LICENSE IN LICENSE.md",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@anthropic-ai/claude-code-win32-arm64": {
|
||||
"version": "2.1.278",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-win32-arm64/-/claude-code-win32-arm64-2.1.278.tgz",
|
||||
"integrity": "sha512-SQf1ZDCenMPc0730RaZsIR8n15x0kdYiwGsL5QBSIcFjxRpT5gnonf5rZ55aeaO9g2g/ZU2CT99RgqMxy5y0Hg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "SEE LICENSE IN LICENSE.md",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/@anthropic-ai/claude-code-win32-x64": {
|
||||
"version": "2.1.278",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-win32-x64/-/claude-code-win32-x64-2.1.278.tgz",
|
||||
"integrity": "sha512-8gwotnfAqgtuayi0m/Ajy5k+fochciv9Erdn7iqaCHM321cz5nOnJy0yCCkk4DsT0nG7kq67dOPXE9N4jdPz7Q==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "SEE LICENSE IN LICENSE.md",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/@openai/codex": {
|
||||
"version": "0.155.1",
|
||||
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1.tgz",
|
||||
"integrity": "sha512-02fAAGyBtlA1zPjEo3kTj/bOSYbPz5DvjLwRZJdV7weFFEDzNFOMjQGmZ/+5CuirYV0hE+AZTrnjzwXYU4AdAQ==",
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"codex": "bin/codex.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@openai/codex-darwin-arm64": "npm:@openai/codex@0.155.1-darwin-arm64",
|
||||
"@openai/codex-darwin-x64": "npm:@openai/codex@0.155.1-darwin-x64",
|
||||
"@openai/codex-linux-arm64": "npm:@openai/codex@0.155.1-linux-arm64",
|
||||
"@openai/codex-linux-x64": "npm:@openai/codex@0.155.1-linux-x64",
|
||||
"@openai/codex-win32-arm64": "npm:@openai/codex@0.155.1-win32-arm64",
|
||||
"@openai/codex-win32-x64": "npm:@openai/codex@0.155.1-win32-x64"
|
||||
}
|
||||
},
|
||||
"node_modules/@openai/codex-darwin-arm64": {
|
||||
"name": "@openai/codex",
|
||||
"version": "0.155.1-darwin-arm64",
|
||||
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-darwin-arm64.tgz",
|
||||
"integrity": "sha512-cYxzGcRRoBrncyHlR8ed4yXwcoVJZC1pipGULSyJkGFKXJw/Uu57BklvzayuAptjJIipamnOk32CfUkk1F0bLw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/@openai/codex-darwin-x64": {
|
||||
"name": "@openai/codex",
|
||||
"version": "0.155.1-darwin-x64",
|
||||
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-darwin-x64.tgz",
|
||||
"integrity": "sha512-FDpc+PdELYlyDnhd76Ckm6jNLF+1n3x34Ygd4QLQger810Vkxx/InQ5LY5jwkecJYKcbvyhMmuxTspaj1dLZrA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/@openai/codex-linux-arm64": {
|
||||
"name": "@openai/codex",
|
||||
"version": "0.155.1-linux-arm64",
|
||||
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-linux-arm64.tgz",
|
||||
"integrity": "sha512-X3fRXm2orhJ3KeB8LgKym4XDUiQaqaOGuaa181bcHTsQI7C8m6tcQQbQsKDzT/2IibikzgYL82jvsgMbq43jww==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/@openai/codex-linux-x64": {
|
||||
"name": "@openai/codex",
|
||||
"version": "0.155.1-linux-x64",
|
||||
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-linux-x64.tgz",
|
||||
"integrity": "sha512-atv3HF0mubqB0J/XkQ2JopqKzXJ+/7aQtTB2MkJ9MrraujMIz8zbCCLylLkN3PzpVGTJzzQFN/wD1oq8oJPJKg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/@openai/codex-win32-arm64": {
|
||||
"name": "@openai/codex",
|
||||
"version": "0.155.1-win32-arm64",
|
||||
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-win32-arm64.tgz",
|
||||
"integrity": "sha512-k5x8VO1aF8Xx/nuh1P31TeBgs11WA6i2GJiHqx5YCndFbWzOzUz6aeBaq9+PO6Qfe/Ivennh3I1FKJBU6Q8mpg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/@openai/codex-win32-x64": {
|
||||
"name": "@openai/codex",
|
||||
"version": "0.155.1-win32-x64",
|
||||
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.155.1-win32-x64.tgz",
|
||||
"integrity": "sha512-MO+cCZrgU0Ec7lJP/5NsTe5obJ9/qtRMkQUK0jYWTY1omxLA3lp5IOD2IAmsejlEJB931XRo51LZ7hl178CDjA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/@standard-schema/spec": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz",
|
||||
"integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@vercel/oidc": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz",
|
||||
"integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">= 20"
|
||||
}
|
||||
},
|
||||
"node_modules/@workflow/serde": {
|
||||
"version": "4.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@workflow/serde/-/serde-4.1.0.tgz",
|
||||
"integrity": "sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/ai": {
|
||||
"version": "7.0.109",
|
||||
"resolved": "https://registry.npmjs.org/ai/-/ai-7.0.109.tgz",
|
||||
"integrity": "sha512-hTnm8K4GIT0R6gHrnDJHr1KRnGVAtFa26mLtjqn/TGy+ufD5Y5XXerTEabLXVgmb9iunWxSFRCPOZUZ/VeXYLQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@ai-sdk/gateway": "4.0.88",
|
||||
"@ai-sdk/provider": "4.0.17",
|
||||
"@ai-sdk/provider-utils": "5.0.45"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"zod": "^3.25.76 || ^4.1.8"
|
||||
}
|
||||
},
|
||||
"node_modules/eventsource-parser": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz",
|
||||
"integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/json-schema": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz",
|
||||
"integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==",
|
||||
"license": "(AFL-2.1 OR BSD-3-Clause)"
|
||||
},
|
||||
"node_modules/undici": {
|
||||
"version": "7.29.1",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz",
|
||||
"integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=20.18.1"
|
||||
}
|
||||
},
|
||||
"node_modules/zod": {
|
||||
"version": "4.6.5",
|
||||
"resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz",
|
||||
"integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==",
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/colinhacks"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
15
tests/e2e/client_apps/package.json
Normal file
15
tests/e2e/client_apps/package.json
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"name": "litellm-e2e-client-apps",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/openai": "4.0.72",
|
||||
"@anthropic-ai/claude-code": "2.1.278",
|
||||
"@openai/codex": "0.155.1",
|
||||
"ai": "7.0.109",
|
||||
"zod": "4.6.5"
|
||||
}
|
||||
}
|
||||
21
tests/e2e/client_apps/spend_rows.py
Normal file
21
tests/e2e/client_apps/spend_rows.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
"""The proxy-side half of every client_apps cell: the spend rows the proxy wrote
|
||||
for the key, each a success with real spend and a request body the client sent
|
||||
with `stream: true`. Every client here makes at least two calls per tool turn
|
||||
(the tool call, then the reply), so the poll waits for two rows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from client_apps_models import RecordedRequest
|
||||
from proxy_client import ProxyClient, SpendLogRow
|
||||
|
||||
|
||||
def recorded_request(row: SpendLogRow) -> RecordedRequest:
|
||||
return RecordedRequest.model_validate(row.proxy_server_request)
|
||||
|
||||
|
||||
def streamed_tool_turn_rows(proxy: ProxyClient, key: str) -> list[SpendLogRow]:
|
||||
rows = proxy.poll_logs_for_key(key, min_rows=2)
|
||||
assert all(row.status == "success" for row in rows), rows
|
||||
assert all(row.spend is not None and row.spend > 0 for row in rows), rows
|
||||
assert all(recorded_request(row).stream for row in rows), rows
|
||||
return rows
|
||||
74
tests/e2e/client_apps/test_claude_code_cli_e2e.py
Normal file
74
tests/e2e/client_apps/test_claude_code_cli_e2e.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
"""The Claude Code CLI, pinned in this suite's package.json, runs one headless
|
||||
turn against the proxy on a virtual key: it streams over `/v1/messages`, calls
|
||||
the Bash tool, reads the tool result, and answers. Client side the stream-json
|
||||
events carry the tool_use, the tool_result, the streamed deltas, and the final
|
||||
result; proxy side the spend rows carry real spend and `stream: true`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from client_apps_client import ClientAppsClient, unwrap_run
|
||||
from client_apps_models import (
|
||||
ClaudeAssistantEvent,
|
||||
ClaudeResultEvent,
|
||||
ClaudeStreamEvent,
|
||||
ClaudeUserEvent,
|
||||
ContentBlockDelta,
|
||||
ToolResultBlock,
|
||||
ToolUseBlock,
|
||||
)
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL
|
||||
from spend_rows import streamed_tool_turn_rows
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
PROMPT: Final = "Use the Bash tool to run the command `echo pong` and report what it printed."
|
||||
|
||||
|
||||
@pytest.mark.covers("other.client_apps.claude_code_cli.tool_turn_streams")
|
||||
def test_claude_code_cli_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str) -> None:
|
||||
events = unwrap_run(
|
||||
client.run_claude_code(
|
||||
key=scoped_key, model=CHEAP_ANTHROPIC_MODEL, prompt=PROMPT, allowed_tool="Bash(echo pong)"
|
||||
)
|
||||
)
|
||||
|
||||
tool_uses = [
|
||||
block
|
||||
for event in events
|
||||
if isinstance(event, ClaudeAssistantEvent) and isinstance(event.message.content, list)
|
||||
for block in event.message.content
|
||||
if isinstance(block, ToolUseBlock)
|
||||
]
|
||||
assert [block.name for block in tool_uses] == ["Bash"], events
|
||||
|
||||
tool_results = [
|
||||
block
|
||||
for event in events
|
||||
if isinstance(event, ClaudeUserEvent) and isinstance(event.message.content, list)
|
||||
for block in event.message.content
|
||||
if isinstance(block, ToolResultBlock)
|
||||
]
|
||||
assert len(tool_results) == 1, events
|
||||
assert not tool_results[0].is_error, tool_results[0]
|
||||
assert "pong" in json.dumps(tool_results[0].content), tool_results[0]
|
||||
|
||||
delta_types = [
|
||||
event.event.delta.type
|
||||
for event in events
|
||||
if isinstance(event, ClaudeStreamEvent) and isinstance(event.event, ContentBlockDelta)
|
||||
]
|
||||
assert "input_json_delta" in delta_types, delta_types
|
||||
assert "text_delta" in delta_types, delta_types
|
||||
|
||||
results = [event for event in events if isinstance(event, ClaudeResultEvent)]
|
||||
assert len(results) == 1, events
|
||||
assert results[0].subtype == "success", results[0]
|
||||
assert not results[0].is_error, results[0]
|
||||
assert results[0].result is not None and "pong" in results[0].result, results[0]
|
||||
|
||||
streamed_tool_turn_rows(client.proxy, scoped_key)
|
||||
43
tests/e2e/client_apps/test_codex_cli_e2e.py
Normal file
43
tests/e2e/client_apps/test_codex_cli_e2e.py
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
"""The Codex CLI, pinned in this suite's package.json, runs one headless
|
||||
`codex exec --json` turn against the proxy on a virtual key through a custom
|
||||
model provider (`wire_api = "responses"`, so `/v1/responses`): it runs a shell
|
||||
command, reads its output, and answers. Client side the JSONL events carry the
|
||||
completed command execution and the agent message; proxy side the spend rows
|
||||
carry real spend and `stream: true` (Codex always streams the Responses API)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from client_apps_client import ClientAppsClient, unwrap_run
|
||||
from client_apps_models import CodexAgentMessage, CodexCommandExecution, CodexItemCompleted, CodexTurnCompleted
|
||||
from e2e_config import CHEAP_OPENAI_MODEL
|
||||
from spend_rows import streamed_tool_turn_rows
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
PROMPT: Final = "Run the shell command `echo pong` and reply with exactly what it printed."
|
||||
|
||||
|
||||
@pytest.mark.covers("other.client_apps.codex_cli.tool_turn_streams")
|
||||
def test_codex_cli_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str) -> None:
|
||||
events = unwrap_run(client.run_codex(key=scoped_key, model=CHEAP_OPENAI_MODEL, prompt=PROMPT))
|
||||
|
||||
completed_items = [event.item for event in events if isinstance(event, CodexItemCompleted)]
|
||||
commands = [item for item in completed_items if isinstance(item, CodexCommandExecution)]
|
||||
assert len(commands) == 1, events
|
||||
assert commands[0].status == "completed", commands[0]
|
||||
assert commands[0].exit_code == 0, commands[0]
|
||||
assert "pong" in commands[0].aggregated_output, commands[0]
|
||||
|
||||
messages = [item for item in completed_items if isinstance(item, CodexAgentMessage)]
|
||||
assert len(messages) == 1, events
|
||||
assert "pong" in messages[0].text, messages[0]
|
||||
|
||||
turns = [event for event in events if isinstance(event, CodexTurnCompleted)]
|
||||
assert len(turns) == 1, events
|
||||
assert turns[0].usage.output_tokens > 0, turns[0]
|
||||
|
||||
streamed_tool_turn_rows(client.proxy, scoped_key)
|
||||
52
tests/e2e/client_apps/test_vercel_ai_sdk_e2e.py
Normal file
52
tests/e2e/client_apps/test_vercel_ai_sdk_e2e.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
"""The Vercel AI SDK (`ai` + `@ai-sdk/openai`, pinned in this suite's
|
||||
package.json) runs one `streamText` turn against the proxy on a virtual key,
|
||||
once over `/v1/responses` and once over `/v1/chat/completions`: the model calls
|
||||
a local tool whose result is a secret word only that tool knows, then answers
|
||||
with the word. Client side the summary carries the streamed text deltas, the
|
||||
tool call, the tool result, and the final text; proxy side the spend rows carry
|
||||
real spend, `stream: true`, and the request body shape of the API used."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from client_apps_client import ClientAppsClient, unwrap_run
|
||||
from client_apps_models import VercelApi
|
||||
from e2e_config import CHEAP_OPENAI_MODEL, unique_marker
|
||||
from spend_rows import recorded_request, streamed_tool_turn_rows
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api",
|
||||
[
|
||||
pytest.param(
|
||||
"responses", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_responses.tool_turn_streams")
|
||||
),
|
||||
pytest.param(
|
||||
"chat", marks=pytest.mark.covers("other.client_apps.vercel_ai_sdk_chat_completions.tool_turn_streams")
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_vercel_ai_sdk_streams_a_tool_turn(client: ClientAppsClient, scoped_key: str, api: VercelApi) -> None:
|
||||
secret_word = f"pong-{unique_marker()}"
|
||||
turn = unwrap_run(
|
||||
client.run_vercel_ai_sdk(key=scoped_key, model=CHEAP_OPENAI_MODEL, api=api, secret_word=secret_word)
|
||||
)
|
||||
|
||||
assert turn.errors == [], turn
|
||||
assert [call.tool_name for call in turn.tool_calls] == ["reveal_secret_word"], turn
|
||||
assert [result.output.word for result in turn.tool_results] == [secret_word], turn
|
||||
assert secret_word in turn.text, turn
|
||||
assert turn.text_deltas >= 2, turn
|
||||
assert turn.steps == 2, turn
|
||||
assert turn.finish_reason == "stop", turn
|
||||
|
||||
rows = streamed_tool_turn_rows(client.proxy, scoped_key)
|
||||
bodies = [recorded_request(row) for row in rows]
|
||||
match api:
|
||||
case "responses":
|
||||
assert all(body.input is not None for body in bodies), rows
|
||||
case "chat":
|
||||
assert all(body.messages is not None for body in bodies), rows
|
||||
78
tests/e2e/client_apps/vercel_ai_turn.mjs
Normal file
78
tests/e2e/client_apps/vercel_ai_turn.mjs
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
import { createOpenAI } from "@ai-sdk/openai";
|
||||
import { stepCountIs, streamText, tool } from "ai";
|
||||
import { z } from "zod";
|
||||
|
||||
const required = (name) => {
|
||||
const value = process.env[name];
|
||||
if (!value) {
|
||||
throw new Error(`${name} must be set`);
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
const proxyUrl = required("LITELLM_PROXY_URL");
|
||||
const apiKey = required("LITELLM_API_KEY");
|
||||
const modelId = required("E2E_MODEL");
|
||||
const api = required("E2E_API");
|
||||
const secretWord = required("E2E_SECRET_WORD");
|
||||
|
||||
const openai = createOpenAI({ baseURL: `${proxyUrl}/v1`, apiKey });
|
||||
const modelByApi = {
|
||||
responses: () => openai.responses(modelId),
|
||||
chat: () => openai.chat(modelId),
|
||||
};
|
||||
if (!(api in modelByApi)) {
|
||||
throw new Error(`E2E_API must be one of ${Object.keys(modelByApi).join(", ")}, got ${api}`);
|
||||
}
|
||||
|
||||
const result = streamText({
|
||||
model: modelByApi[api](),
|
||||
prompt:
|
||||
"Call the reveal_secret_word tool once, then reply with exactly the word it returns and nothing else.",
|
||||
tools: {
|
||||
reveal_secret_word: tool({
|
||||
description: "Reveals the secret word the user is waiting for.",
|
||||
inputSchema: z.object({
|
||||
reason: z.string().describe("Why the word is needed"),
|
||||
}),
|
||||
execute: async () => ({ word: secretWord }),
|
||||
}),
|
||||
},
|
||||
stopWhen: stepCountIs(2),
|
||||
});
|
||||
|
||||
const summary = {
|
||||
api,
|
||||
model: modelId,
|
||||
text_deltas: 0,
|
||||
tool_calls: [],
|
||||
tool_results: [],
|
||||
errors: [],
|
||||
};
|
||||
for await (const part of result.fullStream) {
|
||||
switch (part.type) {
|
||||
case "text-delta":
|
||||
summary.text_deltas += 1;
|
||||
break;
|
||||
case "tool-call":
|
||||
summary.tool_calls.push({ tool_name: part.toolName, input: part.input });
|
||||
break;
|
||||
case "tool-result":
|
||||
summary.tool_results.push({ tool_name: part.toolName, output: part.output });
|
||||
break;
|
||||
case "error":
|
||||
summary.errors.push(String(part.error?.message ?? part.error));
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
process.stdout.write(
|
||||
JSON.stringify({
|
||||
...summary,
|
||||
text: await result.text,
|
||||
steps: (await result.steps).length,
|
||||
finish_reason: await result.finishReason,
|
||||
}),
|
||||
);
|
||||
|
|
@ -54,3 +54,8 @@
|
|||
|
||||
- {id: other.auth.jwt.wrong_issuer_denied, module: other, tier: P0, area: auth, assertions: [wrong_issuer_denied], source: "auth/handle_jwt.py", rationale: "A signed token with the correct audience and an unexpected issuer is rejected"}
|
||||
- {id: other.auth.jwt.wrong_audience_denied, module: other, tier: P0, area: auth, assertions: [wrong_audience_denied], source: "auth/handle_jwt.py", rationale: "A signed token from the trusted issuer intended for another app is rejected"}
|
||||
|
||||
- {id: other.client_apps.claude_code_cli.tool_turn_streams, module: other, tier: P1, area: client_apps, assertions: [tool_turn_streams], source: "anthropic_endpoints/endpoints.py /v1/messages", rationale: "The Claude Code CLI on a virtual key completes one streamed turn with a Bash tool call, and the proxy bills every streamed call"}
|
||||
- {id: other.client_apps.codex_cli.tool_turn_streams, module: other, tier: P1, area: client_apps, assertions: [tool_turn_streams], source: "response_api_endpoints/endpoints.py /v1/responses", rationale: "The Codex CLI on a virtual key completes one streamed turn with a shell command, and the proxy bills every streamed call"}
|
||||
- {id: other.client_apps.vercel_ai_sdk_responses.tool_turn_streams, module: other, tier: P1, area: client_apps, assertions: [tool_turn_streams], source: "response_api_endpoints/endpoints.py /v1/responses", rationale: "A Vercel AI SDK streamText turn over the Responses API on a virtual key calls a tool and answers with its result, and the proxy bills every streamed call"}
|
||||
- {id: other.client_apps.vercel_ai_sdk_chat_completions.tool_turn_streams, module: other, tier: P1, area: client_apps, assertions: [tool_turn_streams], source: "proxy_server.py /v1/chat/completions", rationale: "A Vercel AI SDK streamText turn over chat completions on a virtual key calls a tool and answers with its result, and the proxy bills every streamed call"}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue