test(e2e): add a Codex compatibility matrix suite

Add tests/e2e/client_apps, a compatibility matrix for coding-agent CLIs
other than Claude Code, starting with Codex 0.164.0 pinned through npm.
Each of the 36 cells drives the real codex exec against the proxy for one
feature and provider column, records a pass or fail per model, and the
typed matrix builder turns the results into a matrix and flags cells that
went from green to red. The harness parts are covered in
tests/e2e_harness/client_apps
This commit is contained in:
mateo-berri 2026-10-30 09:33:07 -07:00
parent 43c07a0f12
commit 3acdf171ee
79 changed files with 3277 additions and 2 deletions

View file

@ -49,6 +49,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
- `secret_manager/` - the gateway's `key_management_system` against a real secret manager: deployment keys resolved from it (`os.environ/<name>` where the name exists only in the manager) and virtual keys written to and deleted from it. The tests are backend-agnostic and each backend is its own lane, because the setting is global to the proxy: `E2E_SECRET_MANAGER=<system>` opts in and picks the backend from `secret_backends.BACKENDS`, the proxy is booted from `gateway/secret_manager_<system>_ci_config.yml` against the live manager, and the tests reach that manager through the backend's `SecretStore` (`secret_store_<system>.py`). A test needing something not every backend does carries `requires_capability(...)` and is deselected on lanes that lack it. `secret_manager/backend.sh up <system>` runs a backend in Docker and writes the proxy's and the tests' env. Marked `secret_manager`, deselected unless `E2E_SECRET_MANAGER` is set, and kept out of the per-PR selector. Backends today: `hashicorp_vault` and `cyberark` (CyberArk Conjur, which cannot delete, so the delete test is Vault-only)
- `gateway/` - proxy configuration only (`litellm-config.yml`). No tests
- `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke
- `client_apps/` - compatibility matrices for coding-agent CLIs other than Claude Code, one subfolder per client (`codex/` today). A client folder holds `manifest.yaml` (its provider columns and feature rows), `proxy_models.yaml` (the deployments each column registers through `/model/new` on first use and deletes at session end) and one cell per `<feature>/test_<provider>.py`. A cell drives the real CLI against the proxy with a scoped key for every model in its column and reports one tagged-union outcome per model through the `compat_result` fixture, and the matrix marks the cell green only when every model passed. Cells of one column take turns across xdist workers through a file lock, since a provider's rate limit is shared by the whole column, while different columns run in parallel. The CLIs are pinned in `client_apps/package.json` and its lockfile, the runner image installs them with `npm ci`, and the driver runs `node_modules/.bin/<cli>` before falling back to the one on `PATH`. Nothing here imports `claude_code/`. To run it locally, run `npm ci` in `tests/e2e/client_apps`, then `uv run pytest tests/e2e/client_apps -n 8` with `LITELLM_PROXY_URL` and `LITELLM_MASTER_KEY` set. Point pytest at `tests/e2e/client_apps` or below, because `client_apps/conftest.py` is what writes the results to `<CLIENT>_COMPAT_RESULTS_PATH` (default `<client>-compat-results.json`, merged across xdist workers). `cd tests/e2e && python -m client_apps.matrix_builder build --manifest client_apps/codex/manifest.yaml --results <results> --out <matrix>` turns them into the matrix, and `regressions --old <matrix> --new <matrix>` exits 3 when a green cell turned red. The builder, collector, parser and driver are covered by `tests/e2e_harness/client_apps/`
- `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream, see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport. The Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`. Its specs follow the dashboard's rule against testing what shadcn and Base UI already guarantee, in `ui/litellm-dashboard/AGENTS.md`
## MCP suite: real Datadog only

View file

View file

View file

@ -0,0 +1,27 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, basic_messaging, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("azure")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.azure_openai.basic.stream.codex_basic_messaging")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.AZURE,),
models=MODELS,
mode=Mode.STREAM,
)
)
def test_basic_messaging_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, basic_messaging)

View file

@ -0,0 +1,27 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, basic_messaging, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("bedrock_mantle")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.bedrock_mantle.basic.stream.codex_basic_messaging")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.BEDROCK_MANTLE,),
models=MODELS,
mode=Mode.STREAM,
)
)
def test_basic_messaging_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, basic_messaging)

View file

@ -0,0 +1,27 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, basic_messaging, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("openai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.openai.basic.stream.codex_basic_messaging")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.OPENAI,),
models=MODELS,
mode=Mode.STREAM,
)
)
def test_basic_messaging_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, basic_messaging)

View file

@ -0,0 +1,27 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, basic_messaging, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("vertex_ai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.vertex.basic.stream.codex_basic_messaging")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.VERTEX_AI,),
models=MODELS,
mode=Mode.STREAM,
)
)
def test_basic_messaging_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, basic_messaging)

View file

@ -0,0 +1,210 @@
from __future__ import annotations
import json
import os
import shutil
import signal
import subprocess
from collections.abc import Callable, Mapping
from dataclasses import dataclass, field
from enum import StrEnum
from itertools import chain
from pathlib import Path
from types import MappingProxyType
from typing import Final
from e2e_metadata import step
from client_apps.codex.events import Transcript, parse_transcript
CLIENT_APPS_DIR: Final = Path(__file__).resolve().parents[1]
LOCAL_BINARY: Final = CLIENT_APPS_DIR / "node_modules" / ".bin" / "codex"
TIMEOUT_ENV: Final = "LITELLM_COMPAT_CODEX_TIMEOUT_SECONDS"
DEFAULT_TIMEOUT_SECONDS: Final = 300.0
LITELLM_PROVIDER_KEY_ENV: Final = "LITELLM_API_KEY"
OPENAI_PROVIDER_KEY_ENV: Final = "CODEX_API_KEY"
SANDBOX_MODE: Final = "danger-full-access"
STDERR_TAIL_CHARS: Final = 600
class Wiring(StrEnum):
LITELLM_PROVIDER = "litellm_provider"
OPENAI_BASE_URL = "openai_base_url"
@dataclass(frozen=True, slots=True)
class Gateway:
base_url: str
api_key: str = field(repr=False)
@dataclass(frozen=True, slots=True)
class CodexSession:
root: Path
@property
def codex_home(self) -> Path:
return self.root / "codex_home"
@property
def home(self) -> Path:
return self.root / "home"
@property
def workdir(self) -> Path:
return self.root / "workdir"
def new_session(root: Path) -> CodexSession:
session: Final = CodexSession(root=root)
for directory in (session.codex_home, session.home, session.workdir):
directory.mkdir(parents=True, exist_ok=True)
return session
@dataclass(frozen=True, slots=True)
class CodexTurn:
model: str
prompt: str
options: tuple[str, ...] = ()
resume_thread: str | None = None
wiring: Wiring = Wiring.LITELLM_PROVIDER
@dataclass(frozen=True, slots=True)
class Finished:
exit_code: int
transcript: Transcript
stderr: str
@dataclass(frozen=True, slots=True)
class TimedOut:
seconds: float
transcript: Transcript
stderr: str
type CodexRun = Finished | TimedOut
def resolve_binary(local: Path = LOCAL_BINARY, which: Callable[[str], str | None] = shutil.which) -> Path | None:
if local.is_file():
return local
on_path: Final = which("codex")
return Path(on_path) if on_path is not None else None
def timeout_seconds(environ: Mapping[str, str]) -> float:
return float(environ.get(TIMEOUT_ENV, DEFAULT_TIMEOUT_SECONDS))
def _toml_string(value: str) -> str:
return json.dumps(value)
def _overrides(settings: Mapping[str, str]) -> tuple[str, ...]:
return tuple(chain.from_iterable(("-c", f"{key}={_toml_string(value)}") for key, value in settings.items()))
def wiring_args(wiring: Wiring, base_url: str) -> tuple[str, ...]:
match wiring:
case Wiring.LITELLM_PROVIDER:
return _overrides(
MappingProxyType(
{
"model_provider": "litellm",
"model_providers.litellm.name": "LiteLLM",
"model_providers.litellm.base_url": f"{base_url}/v1",
"model_providers.litellm.env_key": LITELLM_PROVIDER_KEY_ENV,
"model_providers.litellm.wire_api": "responses",
}
)
)
case Wiring.OPENAI_BASE_URL:
return _overrides(MappingProxyType({"openai_base_url": f"{base_url}/v1"}))
def _key_env(wiring: Wiring) -> str:
match wiring:
case Wiring.LITELLM_PROVIDER:
return LITELLM_PROVIDER_KEY_ENV
case Wiring.OPENAI_BASE_URL:
return OPENAI_PROVIDER_KEY_ENV
def command(binary: Path, gateway: Gateway, session: CodexSession, turn: CodexTurn) -> tuple[str, ...]:
resume: Final = ("resume", turn.resume_thread) if turn.resume_thread is not None else ()
return (
str(binary),
"exec",
"--json",
"--skip-git-repo-check",
"--ignore-user-config",
"-C",
str(session.workdir),
"-m",
turn.model,
"-s",
SANDBOX_MODE,
*_overrides(MappingProxyType({"approval_policy": "never"})),
*wiring_args(turn.wiring, gateway.base_url),
*turn.options,
*resume,
"--",
turn.prompt,
)
def environment(gateway: Gateway, session: CodexSession, wiring: Wiring) -> Mapping[str, str]:
return MappingProxyType(
{
"PATH": os.environ.get("PATH", os.defpath),
"HOME": str(session.home),
"CODEX_HOME": str(session.codex_home),
_key_env(wiring): gateway.api_key,
}
)
def _turn_files(session: CodexSession) -> tuple[Path, Path]:
turn: Final = len(tuple(session.root.glob("turn-*.jsonl"))) + 1
return session.root / f"turn-{turn}.jsonl", session.root / f"turn-{turn}.stderr.txt"
@step("Run codex exec against {turn.model}")
def run_codex(binary: Path, gateway: Gateway, session: CodexSession, turn: CodexTurn, seconds: float) -> CodexRun:
stdout_path, stderr_path = _turn_files(session)
with (
stdout_path.open("w") as stdout,
stderr_path.open("w") as stderr,
subprocess.Popen(
command(binary, gateway, session, turn),
cwd=session.workdir,
env=environment(gateway, session, turn.wiring),
stdin=subprocess.DEVNULL,
stdout=stdout,
stderr=stderr,
start_new_session=True,
) as process,
):
try:
exit_code: Final = process.wait(timeout=seconds)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGKILL)
_ = process.wait()
return TimedOut(
seconds=seconds,
transcript=parse_transcript(stdout_path.read_text()),
stderr=stderr_path.read_text(),
)
return Finished(
exit_code=exit_code, transcript=parse_transcript(stdout_path.read_text()), stderr=stderr_path.read_text()
)
def stderr_tail(stderr: str) -> str:
lines: Final = tuple(
line for line in stderr.splitlines() if line.strip() and not line.startswith("Reading additional input")
)
return "\n".join(lines)[-STDERR_TAIL_CHARS:]

View file

@ -0,0 +1,46 @@
from __future__ import annotations
import fcntl
import tempfile
from collections.abc import Generator, Mapping
from contextlib import contextmanager
from functools import cache
from pathlib import Path
from typing import Final
import yaml
from models import LiteLLMParamsBody
from pydantic import BaseModel, ConfigDict, TypeAdapter
PROXY_MODELS_PATH: Final = Path(__file__).resolve().parent / "proxy_models.yaml"
COLUMN_LOCK_DIR: Final = Path(tempfile.gettempdir()) / "litellm-codex-compat-columns"
class Deployment(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
model_name: str
litellm_params: LiteLLMParamsBody
COLUMNS: Final = TypeAdapter(Mapping[str, tuple[Deployment, ...]])
@cache
def load_columns(path: Path = PROXY_MODELS_PATH) -> Mapping[str, tuple[Deployment, ...]]:
return COLUMNS.validate_python(yaml.safe_load(path.read_text()))
def column_models(provider: str) -> tuple[str, ...]:
return tuple(deployment.model_name for deployment in load_columns()[provider])
@contextmanager
def column_lock(provider: str, lock_dir: Path = COLUMN_LOCK_DIR) -> Generator[None]:
lock_dir.mkdir(parents=True, exist_ok=True)
with (lock_dir / f"{provider}.lock").open("a") as handle:
fcntl.flock(handle, fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(handle, fcntl.LOCK_UN)

View file

@ -0,0 +1,63 @@
from __future__ import annotations
import os
from collections.abc import Iterator
from dataclasses import dataclass
from pathlib import Path
from typing import Final
import pytest
from e2e_config import PROXY_BASE_URL
from proxy_client import ProxyClient
from client_apps.codex.cli_driver import Gateway, resolve_binary, timeout_seconds
from client_apps.codex.columns import column_lock, load_columns
from client_apps.codex.features import CodexCell
from client_apps.codex.registry import ColumnRegistry
from client_apps.compat_collector import CompatRecorder
@dataclass(frozen=True, slots=True)
class CodexSuiteClient:
proxy: ProxyClient
@pytest.fixture(scope="session")
def client(proxy: ProxyClient) -> CodexSuiteClient:
return CodexSuiteClient(proxy=proxy)
@pytest.fixture(scope="session")
def codex_binary() -> Path:
binary: Final = resolve_binary()
if binary is None:
pytest.fail("No codex binary: run `npm ci` in tests/e2e/client_apps or put codex on PATH")
return binary
@pytest.fixture(scope="session")
def column_registry(proxy: ProxyClient) -> Iterator[ColumnRegistry]:
registry: Final = ColumnRegistry(proxy, load_columns())
yield registry
registry.release()
@pytest.fixture
def codex_cell(
compat_result: CompatRecorder,
column_registry: ColumnRegistry,
codex_binary: Path,
scoped_key: str,
tmp_path: Path,
) -> Iterator[CodexCell]:
provider: Final = compat_result.address.provider
errors: Final = column_registry.ensure(provider)
if errors:
pytest.fail("; ".join(f"could not register {error.model_name}: {error.error}" for error in errors))
with column_lock(provider):
yield CodexCell(
binary=codex_binary,
gateway=Gateway(base_url=PROXY_BASE_URL, api_key=scoped_key),
seconds=timeout_seconds(os.environ),
sessions_root=tmp_path,
)

View file

@ -0,0 +1,186 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Annotated, Final, Literal
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
class _Event(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
class AgentMessage(_Event):
type: Literal["agent_message"]
text: str
class Reasoning(_Event):
type: Literal["reasoning"]
text: str = ""
class CommandExecution(_Event):
type: Literal["command_execution"]
command: str
aggregated_output: str = ""
exit_code: int | None = None
status: str
class FileUpdate(_Event):
path: str
kind: str
class FileChange(_Event):
type: Literal["file_change"]
changes: tuple[FileUpdate, ...]
status: str
class WebSearch(_Event):
type: Literal["web_search"]
query: str = ""
class McpToolCall(_Event):
type: Literal["mcp_tool_call"]
server: str = ""
tool: str = ""
status: str = ""
class TodoList(_Event):
type: Literal["todo_list"]
class ItemError(_Event):
type: Literal["error"]
message: str
class UnknownItem(_Event):
type: str
type Item = Annotated[
AgentMessage
| Reasoning
| CommandExecution
| FileChange
| WebSearch
| McpToolCall
| TodoList
| ItemError
| UnknownItem,
Field(union_mode="left_to_right"),
]
class Usage(_Event):
input_tokens: int = 0
cached_input_tokens: int = 0
output_tokens: int = 0
reasoning_output_tokens: int = 0
class ErrorDetail(_Event):
message: str
class ThreadStarted(_Event):
type: Literal["thread.started"]
thread_id: str
class TurnStarted(_Event):
type: Literal["turn.started"]
class TurnCompleted(_Event):
type: Literal["turn.completed"]
usage: Usage
class TurnFailed(_Event):
type: Literal["turn.failed"]
error: ErrorDetail
class ItemEvent(_Event):
type: Literal["item.started", "item.updated", "item.completed"]
item: Item
class StreamError(_Event):
type: Literal["error"]
message: str
class UnknownEvent(_Event):
type: str
type CodexEvent = Annotated[
ThreadStarted | TurnStarted | TurnCompleted | TurnFailed | ItemEvent | StreamError | UnknownEvent,
Field(union_mode="left_to_right"),
]
EVENT: Final = TypeAdapter[CodexEvent](CodexEvent)
@dataclass(frozen=True, slots=True)
class UnparsedLine:
text: str
error: str
def parse_line(line: str) -> CodexEvent | UnparsedLine:
try:
return EVENT.validate_json(line)
except ValidationError as error:
return UnparsedLine(text=line, error=str(error))
@dataclass(frozen=True, slots=True)
class Transcript:
events: tuple[CodexEvent, ...]
unparsed: tuple[UnparsedLine, ...]
@property
def thread_id(self) -> str | None:
return next((event.thread_id for event in self.events if isinstance(event, ThreadStarted)), None)
@property
def completed_items(self) -> tuple[Item, ...]:
return tuple(
event.item for event in self.events if isinstance(event, ItemEvent) and event.type == "item.completed"
)
@property
def final_message(self) -> str | None:
messages: Final = tuple(item.text for item in self.completed_items if isinstance(item, AgentMessage))
return messages[-1] if messages else None
@property
def usage(self) -> Usage | None:
turns: Final = tuple(event.usage for event in self.events if isinstance(event, TurnCompleted))
return turns[-1] if turns else None
@property
def failure(self) -> str | None:
turn_failures: Final = tuple(event.error.message for event in self.events if isinstance(event, TurnFailed))
turn_failure: Final = turn_failures[0] if turn_failures else None
if turn_failure is not None or self.usage is not None:
return turn_failure
stream_errors: Final = tuple(event.message for event in self.events if isinstance(event, StreamError))
return stream_errors[-1] if stream_errors else None
def parse_transcript(stdout: str) -> Transcript:
parsed: Final = tuple(parse_line(line) for line in stdout.splitlines() if line.strip())
return Transcript(
events=tuple(event for event in parsed if not isinstance(event, UnparsedLine)),
unparsed=tuple(event for event in parsed if isinstance(event, UnparsedLine)),
)

View file

@ -0,0 +1,299 @@
from __future__ import annotations
import secrets
import struct
import zlib
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from functools import partial
from itertools import chain
from pathlib import Path
from typing import Final
import pytest
from e2e_config import unique_marker
from pydantic import BaseModel, ConfigDict, ValidationError
from client_apps.codex.cli_driver import (
CodexRun,
CodexSession,
CodexTurn,
Finished,
Gateway,
TimedOut,
Wiring,
new_session,
run_codex,
stderr_tail,
)
from client_apps.codex.events import CommandExecution, FileChange, Transcript, WebSearch
from client_apps.compat_results import Failed, Passed
from client_apps.compat_collector import CompatRecorder
SWATCHES: Final = (("red", (220, 20, 20)), ("green", (20, 160, 20)), ("blue", (20, 40, 220)))
SWATCH_SIDE: Final = 64
EXPECTED_DOCS_DOMAIN: Final = "litellm.ai"
OUTPUT_SCHEMA: Final = (
'{"type": "object", "properties": {"marker": {"type": "string"}, "word_count": {"type": "integer"}}, '
'"required": ["marker", "word_count"], "additionalProperties": false}'
)
@dataclass(frozen=True, slots=True)
class CodexCell:
binary: Path
gateway: Gateway
seconds: float
sessions_root: Path
def session(self, model: str) -> CodexSession:
return new_session(self.sessions_root / model)
def run(self, session: CodexSession, turn: CodexTurn) -> CodexRun:
return run_codex(self.binary, self.gateway, session, turn, self.seconds)
type Verdict = Passed | Failed
type Feature = Callable[[CodexCell, str], Verdict]
type Check = Callable[[Transcript], str | None]
def assert_cell(compat_result: CompatRecorder, cell: CodexCell, models: tuple[str, ...], feature: Feature) -> None:
with ThreadPoolExecutor(max_workers=len(models)) as pool:
verdicts: Final = tuple(pool.map(partial(feature, cell), models))
for model, verdict in zip(models, verdicts, strict=True):
match verdict:
case Passed():
compat_result.passed(model)
case Failed(error=error):
compat_result.failed(model, error)
failures: Final = tuple(
f"{model}: {verdict.error}"
for model, verdict in zip(models, verdicts, strict=True)
if isinstance(verdict, Failed)
)
if failures:
pytest.fail("\n".join(failures), pytrace=False)
def run_error(run: CodexRun) -> str | None:
match run:
case TimedOut(seconds=seconds, stderr=stderr):
return f"codex exec timed out after {seconds:.0f}s; stderr: {stderr_tail(stderr)}"
case Finished(transcript=transcript) if transcript.failure is not None:
return f"turn failed: {transcript.failure}"
case Finished(exit_code=exit_code, stderr=stderr) if exit_code != 0:
return f"codex exec exited {exit_code}; stderr: {stderr_tail(stderr)}"
case Finished(transcript=transcript) if transcript.usage is None:
return "codex exec ended without a turn.completed event"
case Finished():
return None
def verdict(run: CodexRun, check: Check) -> Verdict:
error: Final = run_error(run) or check(run.transcript)
return Failed(error=error) if error else Passed()
def final_message_has(expected: str) -> Check:
def check(transcript: Transcript) -> str | None:
message: Final = transcript.final_message
if message is not None and expected in message.lower():
return None
return f"final message {message!r} lacks {expected!r}"
return check
def all_checks(*checks: Check) -> Check:
def check(transcript: Transcript) -> str | None:
return next((error for error in (each(transcript) for each in checks) if error is not None), None)
return check
def basic_messaging(cell: CodexCell, model: str) -> Verdict:
marker: Final = unique_marker()
prompt: Final = f"Reply with exactly this token and nothing else: {marker}"
run: Final = cell.run(cell.session(model), CodexTurn(model, prompt))
return verdict(run, final_message_has(marker))
def _echoed_by_shell(marker: str) -> Check:
def check(transcript: Transcript) -> str | None:
commands: Final = tuple(item for item in transcript.completed_items if isinstance(item, CommandExecution))
echoed: Final = any(command.exit_code == 0 and marker in command.aggregated_output for command in commands)
return None if echoed else f"no completed command_execution printed {marker!r}: {commands!r}"
return check
def shell_command(cell: CodexCell, model: str) -> Verdict:
marker: Final = unique_marker()
prompt: Final = f"Run the shell command `echo {marker}` with your shell tool, then reply with its output only."
run: Final = cell.run(cell.session(model), CodexTurn(model, prompt))
return verdict(run, all_checks(_echoed_by_shell(marker), final_message_has(marker)))
def _edited(session: CodexSession, marker: str) -> Check:
target: Final = session.workdir / "notes.txt"
def check(transcript: Transcript) -> str | None:
changes: Final = tuple(item for item in transcript.completed_items if isinstance(item, FileChange))
updates: Final = chain.from_iterable(item.changes for item in changes)
touched: Final = tuple(Path(update.path).name for update in updates)
if target.name not in touched:
return f"no completed file_change item touched {target.name}: {changes!r}"
content: Final = target.read_text() if target.is_file() else None
return None if content is not None and marker in content else f"{target.name} holds {content!r}, not {marker!r}"
return check
def file_edit(cell: CodexCell, model: str) -> Verdict:
marker: Final = unique_marker()
session: Final = cell.session(model)
prompt: Final = (
f"Create a file named notes.txt in the current directory whose only line is {marker}. "
"Use your apply_patch tool to create it, not the shell."
)
return verdict(cell.run(session, CodexTurn(model, prompt)), _edited(session, marker))
def _resumed(cell: CodexCell, model: str, first_prompt: str, second_prompt: str) -> tuple[CodexRun, CodexRun | str]:
session: Final = cell.session(model)
first: Final = cell.run(session, CodexTurn(model, first_prompt))
thread_id: Final = first.transcript.thread_id
first_error: Final = run_error(first)
if first_error is not None or thread_id is None:
return first, f"first turn: {first_error or 'no thread.started event'}"
return first, cell.run(session, CodexTurn(model, second_prompt, resume_thread=thread_id))
def _same_thread(thread_id: str | None) -> Check:
def check(transcript: Transcript) -> str | None:
if transcript.thread_id == thread_id:
return None
return f"resumed thread {transcript.thread_id!r} != {thread_id!r}"
return check
def multi_turn(cell: CodexCell, model: str) -> Verdict:
marker: Final = unique_marker()
first, second = _resumed(
cell,
model,
f"Remember this token for later: {marker}. Reply with OK only.",
"What token did I ask you to remember? Reply with the token only.",
)
if isinstance(second, str):
return Failed(error=second)
return verdict(second, all_checks(_same_thread(first.transcript.thread_id), final_message_has(marker)))
def _cache_read(transcript: Transcript) -> str | None:
cached: Final = transcript.usage.cached_input_tokens if transcript.usage is not None else 0
return None if cached > 0 else f"second turn reported {cached} cached input tokens: {transcript.usage!r}"
def prompt_caching(cell: CodexCell, model: str) -> Verdict:
first_marker: Final = unique_marker()
second_marker: Final = unique_marker()
_, second = _resumed(
cell,
model,
f"Reply with exactly this token and nothing else: {first_marker}",
f"Now reply with exactly this token and nothing else: {second_marker}",
)
if isinstance(second, str):
return Failed(error=second)
return verdict(second, _cache_read)
def _reasoned(transcript: Transcript) -> str | None:
tokens: Final = transcript.usage.reasoning_output_tokens if transcript.usage is not None else 0
return None if tokens > 0 else f"turn reported {tokens} reasoning output tokens: {transcript.usage!r}"
def reasoning_effort(cell: CodexCell, model: str) -> Verdict:
marker: Final = unique_marker()
prompt: Final = (
f"How many prime numbers are there between 1 and 60? Reply with the number only. Request id {marker}."
)
options: Final = ("-c", 'model_reasoning_effort="high"')
return verdict(cell.run(cell.session(model), CodexTurn(model, prompt, options=options)), _reasoned)
def solid_png(rgb: tuple[int, int, int], side: int = SWATCH_SIDE) -> bytes:
row: Final = b"\x00" + bytes(rgb) * side
return b"".join(
(
b"\x89PNG\r\n\x1a\n",
_png_chunk(b"IHDR", struct.pack(">IIBBBBB", side, side, 8, 2, 0, 0, 0)),
_png_chunk(b"IDAT", zlib.compress(row * side)),
_png_chunk(b"IEND", b""),
)
)
def _png_chunk(kind: bytes, data: bytes) -> bytes:
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
def vision(cell: CodexCell, model: str) -> Verdict:
marker: Final = unique_marker()
color, rgb = secrets.choice(SWATCHES)
session: Final = cell.session(model)
image: Final = session.root / "swatch.png"
_ = image.write_bytes(solid_png(rgb))
prompt: Final = f"What single color fills the attached image? Reply with one lowercase word. Request id {marker}."
run: Final = cell.run(session, CodexTurn(model, prompt, options=("-i", str(image))))
return verdict(run, final_message_has(color))
class StructuredAnswer(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid", strict=True)
marker: str
word_count: int
def _structured(marker: str) -> Check:
def check(transcript: Transcript) -> str | None:
try:
answer: Final = StructuredAnswer.model_validate_json(transcript.final_message or "")
except ValidationError as error:
return f"final message {transcript.final_message!r} does not match the output schema: {error}"
expected: Final = StructuredAnswer(marker=marker, word_count=4)
return None if answer == expected else f"structured answer {answer!r} != {expected!r}"
return check
def structured_output(cell: CodexCell, model: str) -> Verdict:
marker: Final = unique_marker()
session: Final = cell.session(model)
schema: Final = session.root / "schema.json"
_ = schema.write_text(OUTPUT_SCHEMA)
prompt: Final = (
f"Put the token {marker} in marker and the number of words in 'the quick brown fox' in word_count."
)
run: Final = cell.run(session, CodexTurn(model, prompt, options=("--output-schema", str(schema))))
return verdict(run, _structured(marker))
def _searched(transcript: Transcript) -> str | None:
searches: Final = tuple(item for item in transcript.completed_items if isinstance(item, WebSearch))
return None if searches else "no completed web_search item"
def web_search(cell: CodexCell, model: str) -> Verdict:
marker: Final = unique_marker()
prompt: Final = (
"Use your web search tool, not the shell, to find the domain of LiteLLM's documentation site. "
f"Reply with the domain only. Request id {marker}."
)
turn: Final = CodexTurn(model, prompt, options=("-c", 'web_search="live"'), wiring=Wiring.OPENAI_BASE_URL)
return verdict(cell.run(cell.session(model), turn), all_checks(_searched, final_message_has(EXPECTED_DOCS_DOMAIN)))

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, file_edit, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("azure")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.azure_openai.tool_use.stream.codex_file_edit")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.AZURE,),
models=MODELS,
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.STREAM,
)
)
def test_file_edit_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, file_edit)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, file_edit, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("bedrock_mantle")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.bedrock_mantle.tool_use.stream.codex_file_edit")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.BEDROCK_MANTLE,),
models=MODELS,
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.STREAM,
)
)
def test_file_edit_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, file_edit)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, file_edit, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("openai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.openai.tool_use.stream.codex_file_edit")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.OPENAI,),
models=MODELS,
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.STREAM,
)
)
def test_file_edit_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, file_edit)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, file_edit, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("vertex_ai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.vertex.tool_use.stream.codex_file_edit")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.VERTEX_AI,),
models=MODELS,
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.STREAM,
)
)
def test_file_edit_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, file_edit)

View file

@ -0,0 +1,29 @@
schema_version: "1"
client: codex
package: "@openai/codex"
providers:
- openai
- azure
- bedrock_mantle
- vertex_ai
features:
- id: basic_messaging
name: Basic messaging
- id: shell_command
name: Shell command
- id: file_edit
name: File edit
- id: multi_turn
name: Multi-turn (resumed thread)
- id: prompt_caching
name: Prompt caching
- id: reasoning_effort
name: Reasoning effort
- id: vision
name: Vision (image input)
- id: structured_output
name: Structured output
- id: web_search
name: Web search

View file

@ -0,0 +1,27 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, multi_turn, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("azure")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.azure_openai.multi_turn.stream.codex_multi_turn")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.AZURE,),
models=MODELS,
mode=Mode.STREAM,
)
)
def test_multi_turn_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, multi_turn)

View file

@ -0,0 +1,27 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, multi_turn, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("bedrock_mantle")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.bedrock_mantle.multi_turn.stream.codex_multi_turn")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.BEDROCK_MANTLE,),
models=MODELS,
mode=Mode.STREAM,
)
)
def test_multi_turn_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, multi_turn)

View file

@ -0,0 +1,27 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, multi_turn, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("openai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.openai.multi_turn.stream.codex_multi_turn")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.OPENAI,),
models=MODELS,
mode=Mode.STREAM,
)
)
def test_multi_turn_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, multi_turn)

View file

@ -0,0 +1,27 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, multi_turn, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("vertex_ai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.vertex.multi_turn.stream.codex_multi_turn")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.VERTEX_AI,),
models=MODELS,
mode=Mode.STREAM,
)
)
def test_multi_turn_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, multi_turn)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, prompt_caching, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("azure")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.azure_openai.prompt_cache_5m.stream.codex_prompt_caching")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.AZURE,),
models=MODELS,
capabilities=(Capability.PROMPT_CACHING,),
mode=Mode.STREAM,
)
)
def test_prompt_caching_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, prompt_caching)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, prompt_caching, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("bedrock_mantle")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.bedrock_mantle.prompt_cache_5m.stream.codex_prompt_caching")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.BEDROCK_MANTLE,),
models=MODELS,
capabilities=(Capability.PROMPT_CACHING,),
mode=Mode.STREAM,
)
)
def test_prompt_caching_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, prompt_caching)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, prompt_caching, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("openai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.openai.prompt_cache_5m.stream.codex_prompt_caching")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.OPENAI,),
models=MODELS,
capabilities=(Capability.PROMPT_CACHING,),
mode=Mode.STREAM,
)
)
def test_prompt_caching_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, prompt_caching)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, prompt_caching, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("vertex_ai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.vertex.prompt_cache_5m.stream.codex_prompt_caching")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.VERTEX_AI,),
models=MODELS,
capabilities=(Capability.PROMPT_CACHING,),
mode=Mode.STREAM,
)
)
def test_prompt_caching_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, prompt_caching)

View file

@ -0,0 +1,47 @@
openai:
- model_name: gpt-6.1-sol-codex-compat-openai
litellm_params:
model: openai/gpt-6.1-sol
api_key: os.environ/OPENAI_API_KEY
- model_name: gpt-6-luna-codex-compat-openai
litellm_params:
model: openai/gpt-6-luna
api_key: os.environ/OPENAI_API_KEY
azure:
- model_name: gpt-6.1-sol-codex-compat-azure
litellm_params:
model: azure/gpt-6.1-sol
api_base: os.environ/AZURE_API_BASE
api_key: os.environ/AZURE_API_KEY
api_version: v1
- model_name: gpt-6-luna-codex-compat-azure
litellm_params:
model: azure/gpt-6-luna
api_base: os.environ/AZURE_API_BASE
api_key: os.environ/AZURE_API_KEY
api_version: v1
bedrock_mantle:
- model_name: gpt-6.1-sol-codex-compat-bedrock-mantle
litellm_params:
model: bedrock_mantle/openai.gpt-6.1-sol
aws_region_name: us-east-1
- model_name: gpt-6-luna-codex-compat-bedrock-mantle
litellm_params:
model: bedrock_mantle/openai.gpt-6-luna
aws_region_name: us-east-1
vertex_ai:
- model_name: gemini-4-argon-codex-compat-vertex-ai
litellm_params:
model: vertex_ai/gemini-4-argon
vertex_project: os.environ/VERTEXAI_PROJECT
vertex_location: global
vertex_credentials: os.environ/VERTEXAI_CREDENTIALS
- model_name: gemini-3.8-flash-codex-compat-vertex-ai
litellm_params:
model: vertex_ai/gemini-3.8-flash
vertex_project: os.environ/VERTEXAI_PROJECT
vertex_location: global
vertex_credentials: os.environ/VERTEXAI_CREDENTIALS

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, reasoning_effort, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("azure")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.azure_openai.thinking.stream.codex_reasoning_effort")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.AZURE,),
models=MODELS,
capabilities=(Capability.REASONING,),
mode=Mode.STREAM,
)
)
def test_reasoning_effort_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, reasoning_effort)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, reasoning_effort, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("bedrock_mantle")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.bedrock_mantle.thinking.stream.codex_reasoning_effort")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.BEDROCK_MANTLE,),
models=MODELS,
capabilities=(Capability.REASONING,),
mode=Mode.STREAM,
)
)
def test_reasoning_effort_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, reasoning_effort)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, reasoning_effort, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("openai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.openai.thinking.stream.codex_reasoning_effort")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.OPENAI,),
models=MODELS,
capabilities=(Capability.REASONING,),
mode=Mode.STREAM,
)
)
def test_reasoning_effort_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, reasoning_effort)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, reasoning_effort, assert_cell
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("vertex_ai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.vertex.thinking.stream.codex_reasoning_effort")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.VERTEX_AI,),
models=MODELS,
capabilities=(Capability.REASONING,),
mode=Mode.STREAM,
)
)
def test_reasoning_effort_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, reasoning_effort)

View file

@ -0,0 +1,47 @@
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
from proxy_client import ProxyClient
from client_apps.codex.columns import Deployment
@dataclass(frozen=True, slots=True)
class RegistrationError:
model_name: str
error: str
def _register(proxy: ProxyClient, deployment: Deployment) -> str | RegistrationError:
try:
return proxy.create_model(deployment.model_name, deployment.litellm_params, provider_live=True)
except AssertionError as error:
return RegistrationError(model_name=deployment.model_name, error=str(error))
class ColumnRegistry:
__slots__ = ("_columns", "_proxy", "_registered")
def __init__(self, proxy: ProxyClient, columns: Mapping[str, tuple[Deployment, ...]]) -> None:
self._proxy: Final = proxy
self._columns: Final = columns
self._registered: Mapping[str, tuple[str | RegistrationError, ...]] = MappingProxyType({})
def ensure(self, provider: str) -> tuple[RegistrationError, ...]:
if provider not in self._registered:
self._registered = MappingProxyType(
{
**self._registered,
provider: tuple(_register(self._proxy, deployment) for deployment in self._columns[provider]),
}
)
return tuple(result for result in self._registered[provider] if isinstance(result, RegistrationError))
def release(self) -> None:
for results in self._registered.values():
for model_id in (result for result in results if isinstance(result, str)):
self._proxy.delete_model(model_id)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, assert_cell, shell_command
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("azure")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.azure_openai.tool_use.stream.codex_shell_command")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.AZURE,),
models=MODELS,
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.STREAM,
)
)
def test_shell_command_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, shell_command)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, assert_cell, shell_command
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("bedrock_mantle")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.bedrock_mantle.tool_use.stream.codex_shell_command")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.BEDROCK_MANTLE,),
models=MODELS,
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.STREAM,
)
)
def test_shell_command_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, shell_command)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, assert_cell, shell_command
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("openai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.openai.tool_use.stream.codex_shell_command")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.OPENAI,),
models=MODELS,
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.STREAM,
)
)
def test_shell_command_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, shell_command)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, assert_cell, shell_command
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("vertex_ai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.vertex.tool_use.stream.codex_shell_command")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.VERTEX_AI,),
models=MODELS,
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.STREAM,
)
)
def test_shell_command_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, shell_command)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, assert_cell, structured_output
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("azure")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.azure_openai.structured_output.stream.codex_structured_output")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.AZURE,),
models=MODELS,
capabilities=(Capability.RESPONSE_SCHEMA,),
mode=Mode.STREAM,
)
)
def test_structured_output_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, structured_output)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, assert_cell, structured_output
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("bedrock_mantle")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.bedrock_mantle.structured_output.stream.codex_structured_output")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.BEDROCK_MANTLE,),
models=MODELS,
capabilities=(Capability.RESPONSE_SCHEMA,),
mode=Mode.STREAM,
)
)
def test_structured_output_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, structured_output)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, assert_cell, structured_output
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("openai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.openai.structured_output.stream.codex_structured_output")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.OPENAI,),
models=MODELS,
capabilities=(Capability.RESPONSE_SCHEMA,),
mode=Mode.STREAM,
)
)
def test_structured_output_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, structured_output)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, assert_cell, structured_output
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("vertex_ai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.vertex.structured_output.stream.codex_structured_output")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.VERTEX_AI,),
models=MODELS,
capabilities=(Capability.RESPONSE_SCHEMA,),
mode=Mode.STREAM,
)
)
def test_structured_output_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, structured_output)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, assert_cell, vision
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("azure")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.azure_openai.vision.stream.codex_vision")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.AZURE,),
models=MODELS,
capabilities=(Capability.VISION,),
mode=Mode.STREAM,
)
)
def test_vision_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, vision)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, assert_cell, vision
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("bedrock_mantle")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.bedrock_mantle.vision.stream.codex_vision")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.BEDROCK_MANTLE,),
models=MODELS,
capabilities=(Capability.VISION,),
mode=Mode.STREAM,
)
)
def test_vision_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, vision)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, assert_cell, vision
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("openai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.openai.vision.stream.codex_vision")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.OPENAI,),
models=MODELS,
capabilities=(Capability.VISION,),
mode=Mode.STREAM,
)
)
def test_vision_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, vision)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, assert_cell, vision
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("vertex_ai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.vertex.vision.stream.codex_vision")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.VERTEX_AI,),
models=MODELS,
capabilities=(Capability.VISION,),
mode=Mode.STREAM,
)
)
def test_vision_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, vision)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, assert_cell, web_search
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("azure")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.azure_openai.web_search.stream.codex_web_search")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.AZURE,),
models=MODELS,
capabilities=(Capability.WEB_SEARCH,),
mode=Mode.STREAM,
)
)
def test_web_search_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, web_search)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, assert_cell, web_search
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("bedrock_mantle")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.bedrock_mantle.web_search.stream.codex_web_search")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.BEDROCK_MANTLE,),
models=MODELS,
capabilities=(Capability.WEB_SEARCH,),
mode=Mode.STREAM,
)
)
def test_web_search_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, web_search)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, assert_cell, web_search
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("openai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.openai.web_search.stream.codex_web_search")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.OPENAI,),
models=MODELS,
capabilities=(Capability.WEB_SEARCH,),
mode=Mode.STREAM,
)
)
def test_web_search_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, web_search)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from client_apps.codex.columns import column_models
from client_apps.codex.features import CodexCell, assert_cell, web_search
from client_apps.compat_collector import CompatRecorder
MODELS: Final = column_models("vertex_ai")
@pytest.mark.e2e
@pytest.mark.covers("llm.responses.vertex.web_search.stream.codex_web_search")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.VERTEX_AI,),
models=MODELS,
capabilities=(Capability.WEB_SEARCH,),
mode=Mode.STREAM,
)
)
def test_web_search_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None:
assert_cell(compat_result, codex_cell, MODELS, web_search)

View file

@ -0,0 +1,139 @@
from __future__ import annotations
import os
from dataclasses import dataclass
from functools import cache
from itertools import chain
from pathlib import Path
from typing import Final
import pytest
from client_apps.compat_results import (
Failed,
NotApplicable,
Outcome,
Passed,
ResultRow,
merged_rows,
results_path,
shard_dir,
write_rows,
)
from client_apps.matrix_builder import Manifest, load_manifest
CLIENT_APPS_DIR: Final = Path(__file__).resolve().parent
ROWS: Final = pytest.StashKey[tuple[ResultRow, ...]]()
NO_RESULT_ERROR: Final = "the test passed without reporting a result"
@dataclass(frozen=True, slots=True)
class CellAddress:
client: str
feature_id: str
provider: str
@cache
def client_manifest(client_dir: Path) -> Manifest:
return load_manifest(client_dir / "manifest.yaml")
def cell_address(test_file: Path, clients_dir: Path = CLIENT_APPS_DIR) -> CellAddress | None:
resolved: Final = test_file.resolve()
if not resolved.is_relative_to(clients_dir):
return None
match resolved.relative_to(clients_dir).parts:
case (client, feature_id, filename) if filename.startswith("test_") and filename.endswith(".py"):
return _manifest_cell(clients_dir / client, feature_id, filename.removeprefix("test_").removesuffix(".py"))
case _:
return None
def _manifest_cell(client_dir: Path, feature_id: str, provider: str) -> CellAddress | None:
if not (client_dir / "manifest.yaml").is_file():
return None
manifest: Final = client_manifest(client_dir)
known_feature: Final = any(feature.id == feature_id for feature in manifest.features)
if not known_feature or provider not in manifest.providers:
return None
return CellAddress(client=client_dir.name, feature_id=feature_id, provider=provider)
def _row(address: CellAddress, model: str | None, outcome: Outcome) -> ResultRow:
return ResultRow(feature_id=address.feature_id, provider=address.provider, model=model, outcome=outcome)
def _record(item: pytest.Item, row: ResultRow) -> None:
item.stash[ROWS] = (*item.stash.get(ROWS, ()), row)
class CompatRecorder:
__slots__ = ("_item", "address")
def __init__(self, item: pytest.Item, address: CellAddress) -> None:
self._item: Final = item
self.address: Final = address
def passed(self, model: str) -> None:
_record(self._item, _row(self.address, model, Passed()))
def failed(self, model: str, error: str) -> None:
_record(self._item, _row(self.address, model, Failed(error=error)))
def not_applicable(self, reason: str) -> None:
_record(self._item, _row(self.address, None, NotApplicable(reason=reason)))
def _fails_cell(report: pytest.TestReport) -> bool:
return report.failed and report.when in ("setup", "call")
def outcome_rows(
address: CellAddress, recorded: tuple[ResultRow, ...], report: pytest.TestReport
) -> tuple[ResultRow, ...]:
if _fails_cell(report) and not any(isinstance(row.outcome, Failed) for row in recorded):
return (_row(address, None, Failed(error=_crash_message(report))),)
if report.passed and report.when == "call" and not recorded:
return (_row(address, None, Failed(error=NO_RESULT_ERROR)),)
return ()
def _crash_message(report: pytest.TestReport) -> str:
lines: Final = tuple(line for line in report.longreprtext.splitlines() if line.strip())
error_lines: Final = tuple(line for line in lines if line.startswith("E "))
return (error_lines or lines or (f"{report.when} failed",))[-1].removeprefix("E ").strip()
def known_clients() -> tuple[str, ...]:
return tuple(sorted(manifest.parent.name for manifest in CLIENT_APPS_DIR.glob("*/manifest.yaml")))
def worker_id() -> str | None:
return os.environ.get("PYTEST_XDIST_WORKER")
def _item_rows(item: pytest.Item, client: str) -> tuple[ResultRow, ...]:
address: Final = cell_address(item.path)
return item.stash.get(ROWS, ()) if address is not None and address.client == client else ()
def client_rows(session: pytest.Session, client: str) -> tuple[ResultRow, ...]:
return tuple(chain.from_iterable(_item_rows(item, client) for item in session.items))
def clear_shards(client: str) -> None:
shards: Final = shard_dir(results_path(client, os.environ))
for stale in shards.glob("*.json"):
stale.unlink()
def publish(client: str, rows: tuple[ResultRow, ...], worker: str | None) -> None:
results: Final = results_path(client, os.environ)
if worker is not None:
if rows:
write_rows(shard_dir(results) / f"{worker}.json", rows)
return
merged: Final = merged_rows(rows, shard_dir(results))
if merged:
write_rows(results, merged)

View file

@ -0,0 +1,74 @@
from __future__ import annotations
from collections.abc import Mapping
from pathlib import Path
from typing import Annotated, Final, Literal
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
class Passed(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
status: Literal["pass"] = "pass"
class Failed(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
status: Literal["fail"] = "fail"
error: str = Field(min_length=1)
class NotApplicable(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
status: Literal["not_applicable"] = "not_applicable"
reason: str = Field(min_length=1)
class NotTested(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
status: Literal["not_tested"] = "not_tested"
type Outcome = Annotated[Passed | Failed | NotApplicable | NotTested, Field(discriminator="status")]
class ResultRow(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
feature_id: str
provider: str
model: str | None
outcome: Outcome
RESULT_ROWS: Final = TypeAdapter(tuple[ResultRow, ...])
def read_rows(path: Path) -> tuple[ResultRow, ...]:
return RESULT_ROWS.validate_json(path.read_bytes())
def write_rows(path: Path, rows: tuple[ResultRow, ...]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
_ = path.write_bytes(RESULT_ROWS.dump_json(sort_rows(rows), indent=2))
def sort_rows(rows: tuple[ResultRow, ...]) -> tuple[ResultRow, ...]:
return tuple(sorted(rows, key=lambda row: (row.feature_id, row.provider, row.model or "")))
def results_path(client: str, environ: Mapping[str, str]) -> Path:
return Path(environ.get(f"{client.upper()}_COMPAT_RESULTS_PATH", f"{client}-compat-results.json"))
def shard_dir(results: Path) -> Path:
return results.with_name(f"{results.name}.shards")
def merged_rows(own: tuple[ResultRow, ...], shards: Path) -> tuple[ResultRow, ...]:
shard_files: Final = tuple(sorted(shards.glob("*.json"))) if shards.is_dir() else ()
return sort_rows((*own, *(row for shard in shard_files for row in read_rows(shard))))

View file

@ -0,0 +1,59 @@
from __future__ import annotations
from collections.abc import Generator
from typing import Final
import pytest
from client_apps.compat_collector import (
ROWS,
CompatRecorder,
cell_address,
clear_shards,
client_rows,
known_clients,
outcome_rows,
publish,
worker_id,
)
@pytest.fixture
def compat_result(request: pytest.FixtureRequest) -> CompatRecorder:
item: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untyped
assert isinstance(item, pytest.Item)
address: Final = cell_address(item.path)
if address is None:
pytest.fail(f"{item.path} is not a <client>/<feature_id>/test_<provider>.py cell of a client manifest")
return CompatRecorder(item, address)
@pytest.hookimpl(tryfirst=True)
def pytest_runtest_setup(item: pytest.Item) -> None:
item.stash[ROWS] = ()
@pytest.hookimpl(wrapper=True)
def pytest_runtest_makereport(
item: pytest.Item, call: pytest.CallInfo[None]
) -> Generator[None, pytest.TestReport, pytest.TestReport]:
report: Final = yield
address: Final = cell_address(item.path)
if address is None:
return report
extra: Final = outcome_rows(address, item.stash.get(ROWS, ()), report)
if extra:
item.stash[ROWS] = (*item.stash.get(ROWS, ()), *extra)
return report
def pytest_sessionstart(session: pytest.Session) -> None:
if worker_id() is not None:
return
for client in known_clients():
clear_shards(client)
def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
for client in known_clients():
publish(client, client_rows(session, client), worker_id())

View file

@ -0,0 +1,215 @@
from __future__ import annotations
import sys
from argparse import ArgumentParser
from collections.abc import Iterator, Mapping
from datetime import datetime, timezone
from importlib.metadata import version
from pathlib import Path
from types import MappingProxyType
from typing import Annotated, Final, Literal
import yaml
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
from client_apps.compat_results import (
Failed,
NotApplicable,
NotTested,
Outcome,
Passed,
ResultRow,
read_rows,
)
CLIENT_APPS_DIR: Final = Path(__file__).resolve().parent
REGRESSION_EXIT_CODE: Final = 3
class _Frozen(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
class ManifestFeature(_Frozen):
id: str
name: str
class Manifest(_Frozen):
schema_version: str
client: str
package: str
providers: tuple[str, ...]
features: tuple[ManifestFeature, ...]
class MatrixFeature(_Frozen):
id: str
name: str
providers: Mapping[str, Outcome]
class Matrix(_Frozen):
schema_version: str
generated_at: str
litellm_version: str
client: str
client_version: str
providers: tuple[str, ...]
features: tuple[MatrixFeature, ...]
class Regression(_Frozen):
feature_id: str
feature_name: str
provider: str
error: str
class _PackageJson(BaseModel):
model_config = ConfigDict(frozen=True)
dependencies: Mapping[str, str]
class BuildArgs(_Frozen):
command: Literal["build"]
manifest: Path
results: Path
out: Path
litellm_version: str | None
class RegressionArgs(_Frozen):
command: Literal["regressions"]
old: Path
new: Path
CLI_ARGS: Final = TypeAdapter[BuildArgs | RegressionArgs](
Annotated[BuildArgs | RegressionArgs, Field(discriminator="command")]
)
REGRESSIONS: Final = TypeAdapter(tuple[Regression, ...])
def load_manifest(path: Path) -> Manifest:
return Manifest.model_validate(yaml.safe_load(path.read_text()))
def pinned_client_version(package: str, package_json: Path = CLIENT_APPS_DIR / "package.json") -> str:
return _PackageJson.model_validate_json(package_json.read_bytes()).dependencies[package]
def aggregate(rows: tuple[ResultRow, ...]) -> Outcome:
tested: Final = tuple(row for row in rows if not isinstance(row.outcome, NotTested))
failures: Final = tuple(_failure(row.model, row.outcome) for row in tested if isinstance(row.outcome, Failed))
if failures:
return Failed(error="; ".join(failures))
if any(isinstance(row.outcome, Passed) for row in tested):
return Passed()
reasons: Final = tuple(row.outcome.reason for row in tested if isinstance(row.outcome, NotApplicable))
if reasons:
return NotApplicable(reason=reasons[0])
return NotTested()
def _failure(model: str | None, outcome: Failed) -> str:
return outcome.error if model is None else f"{model}: {outcome.error}"
def build_matrix(
manifest: Manifest,
rows: tuple[ResultRow, ...],
*,
litellm_version: str,
client_version: str,
generated_at: datetime,
) -> Matrix:
return Matrix(
schema_version=manifest.schema_version,
generated_at=generated_at.isoformat(),
litellm_version=litellm_version,
client=manifest.client,
client_version=client_version,
providers=manifest.providers,
features=tuple(
MatrixFeature(
id=feature.id,
name=feature.name,
providers=MappingProxyType(
{provider: aggregate(_cell_rows(rows, feature.id, provider)) for provider in manifest.providers}
),
)
for feature in manifest.features
),
)
def _cell_rows(rows: tuple[ResultRow, ...], feature_id: str, provider: str) -> tuple[ResultRow, ...]:
return tuple(row for row in rows if row.feature_id == feature_id and row.provider == provider)
def _cells(matrix: Matrix) -> Iterator[tuple[MatrixFeature, str, Outcome]]:
for feature in matrix.features:
for provider, outcome in feature.providers.items():
yield feature, provider, outcome
def find_regressions(old: Matrix, new: Matrix) -> tuple[Regression, ...]:
old_cells: Final = MappingProxyType({(feature.id, provider): outcome for feature, provider, outcome in _cells(old)})
return tuple(
sorted(
(
Regression(feature_id=feature.id, feature_name=feature.name, provider=provider, error=outcome.error)
for feature, provider, outcome in _cells(new)
if isinstance(outcome, Failed) and isinstance(old_cells.get((feature.id, provider)), Passed)
),
key=lambda regression: (regression.feature_id, regression.provider),
)
)
def _build(args: BuildArgs) -> int:
manifest: Final = load_manifest(args.manifest)
matrix: Final = build_matrix(
manifest,
read_rows(args.results),
litellm_version=args.litellm_version or version("litellm"),
client_version=pinned_client_version(manifest.package),
generated_at=datetime.now(timezone.utc),
)
args.out.parent.mkdir(parents=True, exist_ok=True)
_ = args.out.write_text(matrix.model_dump_json(indent=2))
return 0
def _regressions(args: RegressionArgs) -> int:
if not args.old.exists():
return 0
regressions: Final = find_regressions(
Matrix.model_validate_json(args.old.read_bytes()), Matrix.model_validate_json(args.new.read_bytes())
)
_ = sys.stdout.write(REGRESSIONS.dump_json(regressions, indent=2).decode() + "\n")
return REGRESSION_EXIT_CODE if regressions else 0
def main(argv: tuple[str, ...]) -> int:
parser: Final = ArgumentParser(prog="python -m client_apps.matrix_builder")
commands: Final = parser.add_subparsers(dest="command", required=True)
build: Final = commands.add_parser("build")
_ = build.add_argument("--manifest", required=True)
_ = build.add_argument("--results", required=True)
_ = build.add_argument("--out", required=True)
_ = build.add_argument("--litellm-version", dest="litellm_version", default=None)
regressions: Final = commands.add_parser("regressions")
_ = regressions.add_argument("--old", required=True)
_ = regressions.add_argument("--new", required=True)
match CLI_ARGS.validate_python(vars(parser.parse_args(argv))):
case BuildArgs() as build_args:
return _build(build_args)
case RegressionArgs() as regression_args:
return _regressions(regression_args)
if __name__ == "__main__":
raise SystemExit(main(tuple(sys.argv[1:])))

137
tests/e2e/client_apps/package-lock.json generated Normal file
View file

@ -0,0 +1,137 @@
{
"name": "litellm-client-apps-e2e",
"version": "0.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "litellm-client-apps-e2e",
"version": "0.0.0",
"dependencies": {
"@openai/codex": "0.164.0"
}
},
"node_modules/@openai/codex": {
"version": "0.164.0",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0.tgz",
"integrity": "sha512-TWj7UGyPRe3Pj/A3ddNAxs1te3a/W/xAd+J0LHrrFTmsSfahGMOLF/vn88D3QCIgAT3B1rm1y/Z3NYE0rx6cZQ==",
"license": "Apache-2.0",
"bin": {
"codex": "bin/codex.js"
},
"engines": {
"node": ">=16"
},
"optionalDependencies": {
"@openai/codex-darwin-arm64": "npm:@openai/codex@0.164.0-darwin-arm64",
"@openai/codex-darwin-x64": "npm:@openai/codex@0.164.0-darwin-x64",
"@openai/codex-linux-arm64": "npm:@openai/codex@0.164.0-linux-arm64",
"@openai/codex-linux-x64": "npm:@openai/codex@0.164.0-linux-x64",
"@openai/codex-win32-arm64": "npm:@openai/codex@0.164.0-win32-arm64",
"@openai/codex-win32-x64": "npm:@openai/codex@0.164.0-win32-x64"
}
},
"node_modules/@openai/codex-darwin-arm64": {
"name": "@openai/codex",
"version": "0.164.0-darwin-arm64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-darwin-arm64.tgz",
"integrity": "sha512-pDizgOUAtq3/k3AbUFHbaU19WpYKCja+oFpNSTLKrDQj4lTUNSDuTPJip4uQVn2/1EipQdpQrGMOQBVuI1znAA==",
"cpu": [
"arm64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=16"
}
},
"node_modules/@openai/codex-darwin-x64": {
"name": "@openai/codex",
"version": "0.164.0-darwin-x64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-darwin-x64.tgz",
"integrity": "sha512-lIe+udnguPGzBezkgbkj2Eaz0YqOhNXT7jja136Fj0sXfI+UKNS2icmaK14GcoRAgeVEpDTtC1ftamWBPmD1zg==",
"cpu": [
"x64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=16"
}
},
"node_modules/@openai/codex-linux-arm64": {
"name": "@openai/codex",
"version": "0.164.0-linux-arm64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-linux-arm64.tgz",
"integrity": "sha512-LL5iTQjyDL4YRX44EWqhIfEQ5uGnmgxkQBJSxjZSeJ8L1Og83pdlzUJW/PRJ76oaVwOVgh6HXwuPEiBNEHU4lg==",
"cpu": [
"arm64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=16"
}
},
"node_modules/@openai/codex-linux-x64": {
"name": "@openai/codex",
"version": "0.164.0-linux-x64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-linux-x64.tgz",
"integrity": "sha512-XLAoaWsGOT2khzl+P4FYDPTkzAr9W0VFXFaVkqlFdkM0soa1gH/4S0wSD57Ek3HHwkirAEGz2/Mwehseh+l5Mg==",
"cpu": [
"x64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=16"
}
},
"node_modules/@openai/codex-win32-arm64": {
"name": "@openai/codex",
"version": "0.164.0-win32-arm64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-win32-arm64.tgz",
"integrity": "sha512-Zkzt8HNmB/vOnlDo5D7IOlrt5rHaxYQB7iGbNP7bHcXZtrk6gaD7UriwaieGYJpcf8DqAT/0tqzY4xbs8ZwbmQ==",
"cpu": [
"arm64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=16"
}
},
"node_modules/@openai/codex-win32-x64": {
"name": "@openai/codex",
"version": "0.164.0-win32-x64",
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-win32-x64.tgz",
"integrity": "sha512-urWPhwh9YavuoAPIkmVYKBTtd5OtGVL4Ki0Fb28dhK/Mz6Pt4LSy+gbRrZFz5+RXrw9ubvXuXDCcd8beYcWhDg==",
"cpu": [
"x64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=16"
}
}
}
}

View file

@ -0,0 +1,8 @@
{
"name": "litellm-client-apps-e2e",
"version": "0.0.0",
"private": true,
"dependencies": {
"@openai/codex": "0.164.0"
}
}

View file

@ -0,0 +1,40 @@
# Codex compatibility matrix: /v1/responses coverage for the Codex CLI (tests/e2e/client_apps/codex) across
# the four provider columns it drives. Each row is one (feature x provider) cell.
#
# Grammar: llm.responses.<route>.<capability>.stream.codex_<feature_id>
- {id: llm.responses.openai.basic.stream.codex_basic_messaging, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [codex_basic_messaging], source: "codex compat matrix", rationale: "Codex replies through the proxy"}
- {id: llm.responses.azure_openai.basic.stream.codex_basic_messaging, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: basic, streaming: stream, assertions: [codex_basic_messaging], source: "codex compat matrix", rationale: "Codex replies through the proxy"}
- {id: llm.responses.bedrock_mantle.basic.stream.codex_basic_messaging, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: basic, streaming: stream, assertions: [codex_basic_messaging], source: "codex compat matrix", rationale: "Codex replies through the proxy"}
- {id: llm.responses.vertex.basic.stream.codex_basic_messaging, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: basic, streaming: stream, assertions: [codex_basic_messaging], source: "codex compat matrix", rationale: "Codex replies through the proxy"}
- {id: llm.responses.openai.tool_use.stream.codex_shell_command, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: tool_use, streaming: stream, assertions: [codex_shell_command], source: "codex compat matrix", rationale: "Codex runs a shell command with its exec tool through the proxy"}
- {id: llm.responses.azure_openai.tool_use.stream.codex_shell_command, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: tool_use, streaming: stream, assertions: [codex_shell_command], source: "codex compat matrix", rationale: "Codex runs a shell command with its exec tool through the proxy"}
- {id: llm.responses.bedrock_mantle.tool_use.stream.codex_shell_command, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: tool_use, streaming: stream, assertions: [codex_shell_command], source: "codex compat matrix", rationale: "Codex runs a shell command with its exec tool through the proxy"}
- {id: llm.responses.vertex.tool_use.stream.codex_shell_command, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: tool_use, streaming: stream, assertions: [codex_shell_command], source: "codex compat matrix", rationale: "Codex runs a shell command with its exec tool through the proxy"}
- {id: llm.responses.openai.tool_use.stream.codex_file_edit, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: tool_use, streaming: stream, assertions: [codex_file_edit], source: "codex compat matrix", rationale: "Codex edits a file with its apply_patch tool through the proxy"}
- {id: llm.responses.azure_openai.tool_use.stream.codex_file_edit, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: tool_use, streaming: stream, assertions: [codex_file_edit], source: "codex compat matrix", rationale: "Codex edits a file with its apply_patch tool through the proxy"}
- {id: llm.responses.bedrock_mantle.tool_use.stream.codex_file_edit, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: tool_use, streaming: stream, assertions: [codex_file_edit], source: "codex compat matrix", rationale: "Codex edits a file with its apply_patch tool through the proxy"}
- {id: llm.responses.vertex.tool_use.stream.codex_file_edit, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: tool_use, streaming: stream, assertions: [codex_file_edit], source: "codex compat matrix", rationale: "Codex edits a file with apply_patch through the proxy, which Gemini calls through its exec tool"}
- {id: llm.responses.openai.multi_turn.stream.codex_multi_turn, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: multi_turn, streaming: stream, assertions: [codex_multi_turn], source: "codex compat matrix", rationale: "A resumed Codex thread recalls an earlier turn through the proxy"}
- {id: llm.responses.azure_openai.multi_turn.stream.codex_multi_turn, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: multi_turn, streaming: stream, assertions: [codex_multi_turn], source: "codex compat matrix", rationale: "A resumed Codex thread recalls an earlier turn through the proxy"}
- {id: llm.responses.bedrock_mantle.multi_turn.stream.codex_multi_turn, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: multi_turn, streaming: stream, assertions: [codex_multi_turn], source: "codex compat matrix", rationale: "A resumed Codex thread recalls an earlier turn through the proxy"}
- {id: llm.responses.vertex.multi_turn.stream.codex_multi_turn, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: multi_turn, streaming: stream, assertions: [codex_multi_turn], source: "codex compat matrix", rationale: "A resumed Codex thread recalls an earlier turn through the proxy"}
- {id: llm.responses.openai.prompt_cache_5m.stream.codex_prompt_caching, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: prompt_cache_5m, streaming: stream, assertions: [codex_prompt_caching], source: "codex compat matrix", rationale: "The second turn of a Codex thread reports cached input tokens"}
- {id: llm.responses.azure_openai.prompt_cache_5m.stream.codex_prompt_caching, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: prompt_cache_5m, streaming: stream, assertions: [codex_prompt_caching], source: "codex compat matrix", rationale: "The second turn of a Codex thread reports cached input tokens"}
- {id: llm.responses.bedrock_mantle.prompt_cache_5m.stream.codex_prompt_caching, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: prompt_cache_5m, streaming: stream, assertions: [codex_prompt_caching], source: "codex compat matrix", rationale: "The second turn of a Codex thread reports cached input tokens"}
- {id: llm.responses.vertex.prompt_cache_5m.stream.codex_prompt_caching, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: prompt_cache_5m, streaming: stream, assertions: [codex_prompt_caching], source: "codex compat matrix", rationale: "The second turn of a Codex thread reports cached input tokens"}
- {id: llm.responses.openai.thinking.stream.codex_reasoning_effort, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: thinking, streaming: stream, assertions: [codex_reasoning_effort], source: "codex compat matrix", rationale: "Codex model_reasoning_effort=high reaches the model and reasoning tokens come back"}
- {id: llm.responses.azure_openai.thinking.stream.codex_reasoning_effort, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: thinking, streaming: stream, assertions: [codex_reasoning_effort], source: "codex compat matrix", rationale: "Codex model_reasoning_effort=high reaches the model and reasoning tokens come back"}
- {id: llm.responses.bedrock_mantle.thinking.stream.codex_reasoning_effort, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: thinking, streaming: stream, assertions: [codex_reasoning_effort], source: "codex compat matrix", rationale: "Codex model_reasoning_effort=high reaches the model and reasoning tokens come back"}
- {id: llm.responses.vertex.thinking.stream.codex_reasoning_effort, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: thinking, streaming: stream, assertions: [codex_reasoning_effort], source: "codex compat matrix", rationale: "Codex model_reasoning_effort=high reaches the model and reasoning tokens come back"}
- {id: llm.responses.openai.vision.stream.codex_vision, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: vision, streaming: stream, assertions: [codex_vision], source: "codex compat matrix", rationale: "Codex image input reaches the model through the proxy"}
- {id: llm.responses.azure_openai.vision.stream.codex_vision, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: vision, streaming: stream, assertions: [codex_vision], source: "codex compat matrix", rationale: "Codex image input reaches the model through the proxy"}
- {id: llm.responses.bedrock_mantle.vision.stream.codex_vision, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: vision, streaming: stream, assertions: [codex_vision], source: "codex compat matrix", rationale: "Codex image input reaches the model through the proxy"}
- {id: llm.responses.vertex.vision.stream.codex_vision, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: vision, streaming: stream, assertions: [codex_vision], source: "codex compat matrix", rationale: "Codex image input reaches the model through the proxy"}
- {id: llm.responses.openai.structured_output.stream.codex_structured_output, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: structured_output, streaming: stream, assertions: [codex_structured_output], source: "codex compat matrix", rationale: "Codex --output-schema yields JSON matching the schema"}
- {id: llm.responses.azure_openai.structured_output.stream.codex_structured_output, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: structured_output, streaming: stream, assertions: [codex_structured_output], source: "codex compat matrix", rationale: "Codex --output-schema yields JSON matching the schema"}
- {id: llm.responses.bedrock_mantle.structured_output.stream.codex_structured_output, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: structured_output, streaming: stream, assertions: [codex_structured_output], source: "codex compat matrix", rationale: "Codex --output-schema yields JSON matching the schema"}
- {id: llm.responses.vertex.structured_output.stream.codex_structured_output, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: structured_output, streaming: stream, assertions: [codex_structured_output], source: "codex compat matrix", rationale: "Codex --output-schema yields JSON matching the schema"}
- {id: llm.responses.openai.web_search.stream.codex_web_search, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: web_search, streaming: stream, assertions: [codex_web_search], source: "codex compat matrix", rationale: "Codex web search runs through the proxy"}
- {id: llm.responses.azure_openai.web_search.stream.codex_web_search, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: web_search, streaming: stream, assertions: [codex_web_search], source: "codex compat matrix", rationale: "Codex web search runs through the proxy"}
- {id: llm.responses.bedrock_mantle.web_search.stream.codex_web_search, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: web_search, streaming: stream, assertions: [codex_web_search], source: "codex compat matrix", rationale: "Codex web search runs through the proxy"}
- {id: llm.responses.vertex.web_search.stream.codex_web_search, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: web_search, streaming: stream, assertions: [codex_web_search], source: "codex compat matrix", rationale: "Codex web search runs through the proxy"}

View file

@ -52,6 +52,7 @@ LlmRoute = Literal[
"azure_openai",
"bedrock_converse",
"bedrock_invoke",
"bedrock_mantle",
"cohere",
"gemini",
"hosted_vllm",

View file

@ -1,8 +1,8 @@
# e2e harness tests
Tests of the harness under `tests/e2e/` (the transport, the clients, the fixture bundle and replay edge, the stack lock, the IdP launcher, the coverage collector, the JUnit properties and the Claude Code driver), not of the product. They live outside `tests/e2e/` because the Buildkite e2e run copies that folder into the runner image and runs every test in it, so a harness test in there counts as a product test in the nightly numbers. Nothing here needs a proxy, provider keys or the network
Tests of the harness under `tests/e2e/` (the transport, the clients, the fixture bundle and replay edge, the stack lock, the IdP launcher, the coverage collector, the JUnit properties, the Claude Code driver and the client compatibility matrices), not of the product. They live outside `tests/e2e/` because the Buildkite e2e run copies that folder into the runner image and runs every test in it, so a harness test in there counts as a product test in the nightly numbers. Nothing here needs a proxy, provider keys or the network
The layout mirrors `tests/e2e/`: `test_e2e_http.py` covers `tests/e2e/e2e_http.py`, `logging/test_datadog_reader.py` covers `tests/e2e/logging/datadog_reader.py`, and `claude_code/` covers the driver, builder, probe and version resolver. Put a new harness test under the folder that mirrors the suite folder whose module it covers
The layout mirrors `tests/e2e/`: `test_e2e_http.py` covers `tests/e2e/e2e_http.py`, `logging/test_datadog_reader.py` covers `tests/e2e/logging/datadog_reader.py`, `claude_code/` covers the driver, builder, probe and version resolver, and `client_apps/` covers the client compatibility matrices' collector and builder, with `client_apps/codex/` covering the Codex event parser (against `codex exec --json` transcripts recorded from live runs), the CLI driver and the column lock. Put a new harness test under the folder that mirrors the suite folder whose module it covers
Run them from the repo root. `e2e_config` reads `LITELLM_MASTER_KEY` at import and any value will do, the CI lane sets a dummy:

View file

@ -0,0 +1,148 @@
from __future__ import annotations
from pathlib import Path
from typing import Final
import pytest
from client_apps.codex.cli_driver import (
LITELLM_PROVIDER_KEY_ENV,
OPENAI_PROVIDER_KEY_ENV,
TIMEOUT_ENV,
CodexTurn,
Finished,
Gateway,
TimedOut,
Wiring,
command,
environment,
new_session,
resolve_binary,
run_codex,
stderr_tail,
timeout_seconds,
)
GATEWAY: Final = Gateway(base_url="http://127.0.0.1:24001", api_key="sk-scoped")
EMITTING_STUB: Final = """#!/bin/sh
printf '%s\\n' "$@" >&2
echo '{"type":"thread.started","thread_id":"t-1"}'
echo '{"type":"item.completed","item":{"id":"i","type":"agent_message","text":"hi"}}'
echo '{"type":"turn.completed","usage":{"input_tokens":3,"output_tokens":1}}'
exit 4
"""
HANGING_STUB: Final = """#!/bin/sh
echo '{"type":"thread.started","thread_id":"t-2"}'
sleep 30
"""
def _stub(directory: Path, script: str) -> Path:
binary: Final = directory / "codex"
_ = binary.write_text(script)
binary.chmod(0o755)
return binary
def _option(argv: tuple[str, ...], name: str) -> tuple[str, ...]:
return tuple(argv[index + 1] for index, value in enumerate(argv[:-1]) if value == name)
def test_litellm_provider_command_points_codex_at_the_proxy_v1_responses(tmp_path: Path) -> None:
session: Final = new_session(tmp_path)
argv: Final = command(Path("/bin/codex"), GATEWAY, session, CodexTurn("gpt-6.1-sol", "-- say hi"))
assert argv[:2] == ("/bin/codex", "exec")
assert set(_option(argv, "-c")) >= {
'model_provider="litellm"',
'model_providers.litellm.base_url="http://127.0.0.1:24001/v1"',
f'model_providers.litellm.env_key="{LITELLM_PROVIDER_KEY_ENV}"',
'model_providers.litellm.wire_api="responses"',
'approval_policy="never"',
}
assert _option(argv, "-m") == ("gpt-6.1-sol",)
assert _option(argv, "-C") == (str(session.workdir),)
assert argv[-2:] == ("--", "-- say hi")
def test_resume_comes_after_every_exec_option_and_before_the_prompt(tmp_path: Path) -> None:
turn: Final = CodexTurn("gpt-6.1-sol", "again", options=("-i", "img.png"), resume_thread="t-9")
argv: Final = command(Path("codex"), GATEWAY, new_session(tmp_path), turn)
resume_at: Final = argv.index("resume")
assert argv[resume_at : resume_at + 4] == ("resume", "t-9", "--", "again")
assert argv.index("-i") < resume_at
assert max(index for index, value in enumerate(argv) if value == "-c") < resume_at
def test_openai_base_url_wiring_keeps_the_builtin_provider(tmp_path: Path) -> None:
turn: Final = CodexTurn("gpt-6.1-sol", "p", wiring=Wiring.OPENAI_BASE_URL)
overrides: Final = _option(command(Path("codex"), GATEWAY, new_session(tmp_path), turn), "-c")
assert 'openai_base_url="http://127.0.0.1:24001/v1"' in overrides
assert not any(override.startswith("model_provider") for override in overrides)
def test_environment_carries_only_the_session_dirs_and_the_wired_key(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("OPENAI_API_KEY", "sk-leak")
session: Final = new_session(tmp_path)
litellm_env: Final = environment(GATEWAY, session, Wiring.LITELLM_PROVIDER)
openai_env: Final = environment(GATEWAY, session, Wiring.OPENAI_BASE_URL)
assert set(litellm_env) == {"PATH", "HOME", "CODEX_HOME", LITELLM_PROVIDER_KEY_ENV}
assert set(openai_env) == {"PATH", "HOME", "CODEX_HOME", OPENAI_PROVIDER_KEY_ENV}
assert litellm_env[LITELLM_PROVIDER_KEY_ENV] == openai_env[OPENAI_PROVIDER_KEY_ENV] == "sk-scoped"
assert litellm_env["CODEX_HOME"] == str(session.codex_home)
assert litellm_env["HOME"] == str(session.home)
def test_resolve_binary_prefers_the_pinned_local_install(tmp_path: Path) -> None:
local: Final = _stub(tmp_path, "#!/bin/sh\n")
assert resolve_binary(local, which=lambda _: "/usr/bin/codex") == local
assert resolve_binary(tmp_path / "missing", which=lambda _: "/usr/bin/codex") == Path("/usr/bin/codex")
assert resolve_binary(tmp_path / "missing", which=lambda _: None) is None
def test_timeout_seconds_reads_the_env_override() -> None:
assert timeout_seconds({TIMEOUT_ENV: "12.5"}) == 12.5
assert timeout_seconds({}) > 0
def test_run_codex_parses_stdout_and_keeps_each_turn_log(tmp_path: Path) -> None:
binary: Final = _stub(tmp_path, EMITTING_STUB)
session: Final = new_session(tmp_path / "session")
first: Final = run_codex(binary, GATEWAY, session, CodexTurn("m", "p"), 30)
second: Final = run_codex(binary, GATEWAY, session, CodexTurn("m", "q", resume_thread="t-1"), 30)
assert isinstance(first, Finished)
assert first.exit_code == 4
assert first.transcript.thread_id == "t-1"
assert first.transcript.final_message == "hi"
assert first.transcript.usage is not None
assert first.transcript.usage.input_tokens == 3
assert isinstance(second, Finished)
assert "resume\nt-1\n--\nq" in second.stderr
assert sorted(path.name for path in session.root.glob("turn-*")) == [
"turn-1.jsonl",
"turn-1.stderr.txt",
"turn-2.jsonl",
"turn-2.stderr.txt",
]
def test_run_codex_kills_a_hung_turn_and_keeps_what_it_printed(tmp_path: Path) -> None:
run: Final = run_codex(_stub(tmp_path, HANGING_STUB), GATEWAY, new_session(tmp_path / "s"), CodexTurn("m", "p"), 1)
assert isinstance(run, TimedOut)
assert run.transcript.thread_id == "t-2"
def test_stderr_tail_drops_the_stdin_notice_and_keeps_the_end() -> None:
tail: Final = stderr_tail("Reading additional input from stdin...\n\n" + "x" * 2000 + "\nboom")
assert "Reading additional input" not in tail
assert tail.endswith("boom")
assert len(tail) <= 600

View file

@ -0,0 +1,40 @@
from __future__ import annotations
import threading
from itertools import chain
from pathlib import Path
from typing import Final
from client_apps.codex.columns import column_lock, column_models, load_columns
from client_apps.compat_collector import CLIENT_APPS_DIR, client_manifest
def _acquire_in_thread(provider: str, lock_dir: Path) -> threading.Event:
acquired: Final = threading.Event()
def hold() -> None:
with column_lock(provider, lock_dir):
acquired.set()
threading.Thread(target=hold, daemon=True).start()
return acquired
def test_column_lock_lets_one_cell_per_column_run_at_a_time(tmp_path: Path) -> None:
with column_lock("azure", tmp_path):
same_column: Final = _acquire_in_thread("azure", tmp_path)
other_column: Final = _acquire_in_thread("openai", tmp_path)
assert other_column.wait(5)
assert not same_column.wait(0.3)
assert same_column.wait(5)
def test_every_manifest_column_has_deployments_with_unique_names() -> None:
manifest: Final = client_manifest(CLIENT_APPS_DIR / "codex")
names: Final = tuple(chain.from_iterable(column_models(provider) for provider in manifest.providers))
assert set(load_columns()) == set(manifest.providers)
assert all(column_models(provider) for provider in manifest.providers)
assert len(set(names)) == len(names)

View file

@ -0,0 +1,154 @@
from __future__ import annotations
from pathlib import Path
from typing import Final
from client_apps.codex.events import (
CommandExecution,
FileChange,
ItemError,
Reasoning,
Transcript,
UnknownEvent,
UnknownItem,
WebSearch,
parse_transcript,
)
TRANSCRIPTS: Final = Path(__file__).resolve().parent / "transcripts"
def _transcript(name: str) -> Transcript:
return parse_transcript((TRANSCRIPTS / name).read_text())
def test_every_recorded_line_parses() -> None:
unparsed: Final = {path.name: _transcript(path.name).unparsed for path in sorted(TRANSCRIPTS.glob("*.jsonl"))}
assert len(unparsed) == 9
assert all(lines == () for lines in unparsed.values()), unparsed
def test_file_change_lists_the_touched_path_once_completed() -> None:
transcript: Final = _transcript("file_change.jsonl")
changes: Final = tuple(item for item in transcript.completed_items if isinstance(item, FileChange))
assert len(changes) == 1
assert changes[0].status == "completed"
assert [(update.path, update.kind) for update in changes[0].changes] == [
("/tmp/codex-session/workdir/notes.txt", "add")
]
assert transcript.final_message == "Created `notes.txt` containing only `q7w8e9`."
assert transcript.failure is None
def test_shell_command_keeps_exit_code_and_output_of_the_completed_item() -> None:
transcript: Final = _transcript("shell_command.jsonl")
commands: Final = tuple(item for item in transcript.completed_items if isinstance(item, CommandExecution))
assert [(command.exit_code, command.aggregated_output) for command in commands] == [(0, "c887862269b3\n")]
assert transcript.final_message == "c887862269b3"
def test_completed_web_search_carries_its_query() -> None:
searches: Final = tuple(
item for item in _transcript("web_search.jsonl").completed_items if isinstance(item, WebSearch)
)
assert [search.query for search in searches] == ["LiteLLM documentation official"]
def test_web_search_that_only_started_is_not_a_completed_item() -> None:
transcript: Final = _transcript("web_search_never_completed.jsonl")
assert not any(isinstance(item, WebSearch) for item in transcript.completed_items)
assert transcript.final_message == "docs.litellm.ai"
assert transcript.usage is not None
def test_fallback_metadata_warning_is_an_item_not_a_turn_failure() -> None:
transcript: Final = _transcript("fallback_metadata.jsonl")
errors: Final = tuple(item for item in transcript.completed_items if isinstance(item, ItemError))
assert len(errors) == 1
assert "fallback metadata" in errors[0].message
assert transcript.failure is None
assert any(isinstance(item, Reasoning) and item.text for item in transcript.completed_items)
assert transcript.usage is not None
assert transcript.usage.reasoning_output_tokens == 205
assert transcript.final_message == "15"
def test_turn_failed_reports_the_proxy_error_and_no_usage() -> None:
transcript: Final = _transcript("turn_failed.jsonl")
assert transcript.failure is not None
assert "Invalid model name passed in model=gpt-6.1-sol-does-not-exist" in transcript.failure
assert transcript.usage is None
assert transcript.final_message is None
def test_rate_limited_turn_reports_the_final_error_not_a_retry_notice() -> None:
transcript: Final = _transcript("rate_limited.jsonl")
assert transcript.failure is not None
assert transcript.failure.startswith("rate limit exceeded")
assert "Reconnecting" not in transcript.failure
def test_a_turn_that_recovers_after_a_retry_notice_has_no_failure() -> None:
transcript: Final = parse_transcript(
"\n".join(
(
'{"type":"thread.started","thread_id":"t-1"}',
'{"type":"turn.started"}',
'{"type":"error","message":"Reconnecting... 1/5 (rate limit exceeded: 429)"}',
'{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"done"}}',
'{"type":"turn.completed","usage":{"input_tokens":3,"output_tokens":1}}',
)
)
)
assert transcript.failure is None
assert transcript.final_message == "done"
def test_a_stream_that_ends_on_an_error_without_turn_events_is_a_failure() -> None:
transcript: Final = parse_transcript(
'{"type":"thread.started","thread_id":"t-1"}\n{"type":"error","message":"stream disconnected"}\n'
)
assert transcript.failure == "stream disconnected"
def test_resumed_turn_keeps_the_thread_and_reports_cached_input() -> None:
first: Final = _transcript("first_turn.jsonl")
resumed: Final = _transcript("resumed_turn.jsonl")
assert first.thread_id is not None
assert resumed.thread_id == first.thread_id
assert first.usage is not None
assert resumed.usage is not None
assert first.usage.cached_input_tokens == 0
assert resumed.usage.cached_input_tokens == 10870
def test_unknown_shapes_parse_as_unknown_and_garbage_lines_are_kept_aside() -> None:
transcript: Final = parse_transcript(
"\n".join(
(
'{"type":"thread.started","thread_id":"t-1"}',
'{"type":"turn.compacted"}',
'{"type":"item.completed","item":{"id":"item_0","type":"collab_tool_call"}}',
'{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"done"}}',
"not json",
"",
)
)
)
assert transcript.thread_id == "t-1"
assert any(isinstance(event, UnknownEvent) and event.type == "turn.compacted" for event in transcript.events)
assert [item.type for item in transcript.completed_items if isinstance(item, UnknownItem)] == ["collab_tool_call"]
assert transcript.final_message == "done"
assert [line.text for line in transcript.unparsed] == ["not json"]

View file

@ -0,0 +1,6 @@
{"type":"thread.started","thread_id":"01a18d5f-e8d7-7c70-9059-c8a74922438e"}
{"type":"item.completed","item":{"id":"item_0","type":"error","message":"Model metadata for `gemini-4-argon-codex-compat-vertex-ai` not found. Defaulting to fallback metadata; this can degrade performance and cause issues."}}
{"type":"turn.started"}
{"type":"item.completed","item":{"id":"item_1","type":"reasoning","text":"**Enumerating Primes**\n\nI've successfully identified all prime numbers within the 1 to 50 range and meticulously counted each one, establishing their sequence and quantity.\n\n"}}
{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"15"}}
{"type":"turn.completed","usage":{"input_tokens":8517,"cached_input_tokens":0,"cache_write_input_tokens":0,"output_tokens":211,"reasoning_output_tokens":205}}

View file

@ -0,0 +1,7 @@
{"type":"thread.started","thread_id":"01a18d5f-e9af-74e2-9631-f1abd29018b5"}
{"type":"turn.started"}
{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I’ll create `notes.txt` with the requested line."}}
{"type":"item.started","item":{"id":"item_1","type":"file_change","changes":[{"path":"/tmp/codex-session/workdir/notes.txt","kind":"add"}],"status":"in_progress"}}
{"type":"item.completed","item":{"id":"item_1","type":"file_change","changes":[{"path":"/tmp/codex-session/workdir/notes.txt","kind":"add"}],"status":"completed"}}
{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"Created `notes.txt` containing only `q7w8e9`."}}
{"type":"turn.completed","usage":{"input_tokens":21906,"cached_input_tokens":10882,"cache_write_input_tokens":11018,"output_tokens":131,"reasoning_output_tokens":0}}

View file

@ -0,0 +1,4 @@
{"type":"thread.started","thread_id":"01a18d64-83eb-78f3-b6a3-5d7ae37d3d93"}
{"type":"turn.started"}
{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"OK"}}
{"type":"turn.completed","usage":{"input_tokens":10873,"cached_input_tokens":0,"cache_write_input_tokens":10870,"output_tokens":5,"reasoning_output_tokens":0}}

View file

@ -0,0 +1,9 @@
{"type":"thread.started","thread_id":"01a18d84-ca4f-77b1-8367-f3420df1b7a5"}
{"type":"turn.started"}
{"type":"error","message":"Reconnecting... 1/5 (rate limit exceeded: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-6.1-sol for gpt-6.1-sol in eastus2 have exceeded token rate limit.)"}
{"type":"error","message":"Reconnecting... 2/5 (rate limit exceeded: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-6.1-sol for gpt-6.1-sol in eastus2 have exceeded token rate limit.)"}
{"type":"error","message":"Reconnecting... 3/5 (rate limit exceeded: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-6.1-sol for gpt-6.1-sol in eastus2 have exceeded token rate limit.)"}
{"type":"error","message":"Reconnecting... 4/5 (rate limit exceeded: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-6.1-sol for gpt-6.1-sol in eastus2 have exceeded token rate limit.)"}
{"type":"error","message":"Reconnecting... 5/5 (rate limit exceeded: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-6.1-sol for gpt-6.1-sol in eastus2 have exceeded token rate limit.)"}
{"type":"error","message":"rate limit exceeded: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-6.1-sol for gpt-6.1-sol in eastus2 have exceeded token rate limit."}
{"type":"turn.failed","error":{"message":"rate limit exceeded: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-6.1-sol for gpt-6.1-sol in eastus2 have exceeded token rate limit."}}

View file

@ -0,0 +1,4 @@
{"type":"thread.started","thread_id":"01a18d64-83eb-78f3-b6a3-5d7ae37d3d93"}
{"type":"turn.started"}
{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"p3q4r5"}}
{"type":"turn.completed","usage":{"input_tokens":21772,"cached_input_tokens":10870,"cache_write_input_tokens":10896,"output_tokens":15,"reasoning_output_tokens":0}}

View file

@ -0,0 +1,6 @@
{"type":"thread.started","thread_id":"01a18d86-cf06-7c42-8951-618e79c2013e"}
{"type":"turn.started"}
{"type":"item.started","item":{"id":"item_0","type":"command_execution","command":"/bin/bash -lc 'echo c887862269b3'","aggregated_output":"","exit_code":null,"status":"in_progress"}}
{"type":"item.completed","item":{"id":"item_0","type":"command_execution","command":"/bin/bash -lc 'echo c887862269b3'","aggregated_output":"c887862269b3\n","exit_code":0,"status":"completed"}}
{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"c887862269b3"}}
{"type":"turn.completed","usage":{"input_tokens":21984,"cached_input_tokens":10955,"cache_write_input_tokens":11023,"output_tokens":48,"reasoning_output_tokens":0}}

View file

@ -0,0 +1,4 @@
{"type":"thread.started","thread_id":"01a18d67-3419-7430-85ab-dd4795bfd01c"}
{"type":"turn.started"}
{"type":"error","message":"{\"error\":{\"message\":\"/responses: Invalid model name passed in model=gpt-6.1-sol-does-not-exist. Call `/v1/models` to view available models for your key.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"400\",\"provider_specific_fields\":{\"error\":\"/responses: Invalid model name passed in model=gpt-6.1-sol-does-not-exist. Call `/v1/models` to view available models for your key.\"}}}"}
{"type":"turn.failed","error":{"message":"{\"error\":{\"message\":\"/responses: Invalid model name passed in model=gpt-6.1-sol-does-not-exist. Call `/v1/models` to view available models for your key.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"400\",\"provider_specific_fields\":{\"error\":\"/responses: Invalid model name passed in model=gpt-6.1-sol-does-not-exist. Call `/v1/models` to view available models for your key.\"}}}"}}

View file

@ -0,0 +1,7 @@
{"type":"thread.started","thread_id":"01a18d67-686f-7e72-bc9d-f6a24e2320ed"}
{"type":"turn.started"}
{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I’ll look up LiteLLM’s documentation domain."}}
{"type":"item.started","item":{"id":"item_1","type":"web_search","id":"exec-baaa41bf-4055-42d4-8651-fbd2611cf820","query":"","action":{"type":"other"}}}
{"type":"item.completed","item":{"id":"item_1","type":"web_search","id":"exec-baaa41bf-4055-42d4-8651-fbd2611cf820","query":"LiteLLM documentation official","action":{"type":"search","query":"LiteLLM documentation official"}}}
{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"[docs.litellm.ai](https://docs.litellm.ai/)"}}
{"type":"turn.completed","usage":{"input_tokens":30618,"cached_input_tokens":13312,"cache_write_input_tokens":17300,"output_tokens":80,"reasoning_output_tokens":0}}

View file

@ -0,0 +1,7 @@
{"type":"thread.started","thread_id":"01a18d65-9cd0-7cf0-8fdd-ea480a832271"}
{"type":"turn.started"}
{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I’ll look up LiteLLM’s documentation domain."}}
{"type":"item.started","item":{"id":"item_1","type":"web_search","id":"exec-dfea1d72-2e16-4853-a9c5-713fbd2d4a45","query":"","action":{"type":"other"}}}
{"type":"item.started","item":{"id":"item_2","type":"web_search","id":"exec-f0f475b4-d3a1-440d-988a-cad646913023","query":"","action":{"type":"other"}}}
{"type":"item.completed","item":{"id":"item_3","type":"agent_message","text":"docs.litellm.ai"}}
{"type":"turn.completed","usage":{"input_tokens":59995,"cached_input_tokens":43336,"cache_write_input_tokens":16647,"output_tokens":154,"reasoning_output_tokens":14}}

View file

@ -0,0 +1,140 @@
from __future__ import annotations
from pathlib import Path
from typing import Final, Literal
import pytest
from client_apps.compat_collector import NO_RESULT_ERROR, CellAddress, cell_address, clear_shards, outcome_rows, publish
from client_apps.compat_results import Failed, NotApplicable, Passed, ResultRow, read_rows, shard_dir
MANIFEST: Final = """schema_version: "1"
client: fakeclient
package: fake-client
providers: [openai, vertex_ai]
features:
- {id: vision, name: Vision}
"""
ADDRESS: Final = CellAddress(client="fakeclient", feature_id="vision", provider="openai")
RESULTS_ENV: Final = "FAKECLIENT_COMPAT_RESULTS_PATH"
def _clients_dir(root: Path) -> Path:
client: Final = root / "fakeclient"
client.mkdir(parents=True)
_ = (client / "manifest.yaml").write_text(MANIFEST)
return root
def _report(
when: Literal["setup", "call", "teardown"], outcome: Literal["passed", "failed"], longrepr: str | None = None
) -> pytest.TestReport:
return pytest.TestReport(
nodeid="fakeclient/vision/test_openai.py::test_vision_openai",
location=("fakeclient/vision/test_openai.py", 1, "test_vision_openai"),
keywords={},
outcome=outcome,
longrepr=longrepr,
when=when,
)
def _row(model: str | None, outcome: Passed | Failed | NotApplicable) -> ResultRow:
return ResultRow(feature_id="vision", provider="openai", model=model, outcome=outcome)
@pytest.mark.parametrize(
("relative", "expected"),
(
("fakeclient/vision/test_openai.py", ADDRESS),
("fakeclient/vision/test_vertex_ai.py", CellAddress("fakeclient", "vision", "vertex_ai")),
("fakeclient/vision/test_azure.py", None),
("fakeclient/web_search/test_openai.py", None),
("fakeclient/vision/helpers.py", None),
("fakeclient/test_openai.py", None),
("otherclient/vision/test_openai.py", None),
),
)
def test_cell_address_only_maps_manifest_cells(tmp_path: Path, relative: str, expected: CellAddress | None) -> None:
clients: Final = _clients_dir(tmp_path)
assert cell_address(clients / relative, clients) == expected
def test_cell_address_ignores_files_outside_the_clients_dir(tmp_path: Path) -> None:
clients: Final = _clients_dir(tmp_path / "clients")
assert cell_address(tmp_path / "fakeclient" / "vision" / "test_openai.py", clients) is None
def test_a_cell_that_passed_without_reporting_is_recorded_as_failed() -> None:
assert outcome_rows(ADDRESS, (), _report("call", "passed")) == (_row(None, Failed(error=NO_RESULT_ERROR)),)
def test_a_cell_that_crashed_records_the_assertion_line() -> None:
longrepr: Final = "def test():\n> assert x\nE AssertionError: proxy answered 500\n\nfile.py:3: AssertionError"
assert outcome_rows(ADDRESS, (), _report("call", "failed", longrepr)) == (
_row(None, Failed(error="AssertionError: proxy answered 500")),
)
def test_a_setup_failure_is_a_failed_cell() -> None:
rows: Final = outcome_rows(ADDRESS, (), _report("setup", "failed", "could not register gpt-6.1-sol"))
assert rows == (_row(None, Failed(error="could not register gpt-6.1-sol")),)
@pytest.mark.parametrize(
("recorded", "report"),
(
((_row("m", Failed(error="no file_change")),), _report("call", "failed", "Failed: m: no file_change")),
((_row("m", Passed()),), _report("call", "passed")),
((_row(None, NotApplicable(reason="no apply_patch")),), _report("call", "passed")),
((), _report("teardown", "failed", "delete_model warned")),
((), _report("setup", "passed")),
),
)
def test_a_cell_that_reported_its_own_outcome_gets_no_extra_row(
recorded: tuple[ResultRow, ...], report: pytest.TestReport
) -> None:
assert outcome_rows(ADDRESS, recorded, report) == ()
def test_workers_write_shards_and_the_controller_merges_them(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
results: Final = tmp_path / "out" / "fakeclient.json"
monkeypatch.setenv(RESULTS_ENV, str(results))
first: Final = _row("a", Passed())
second: Final = _row("b", Failed(error="boom"))
publish("fakeclient", (second,), "gw1")
publish("fakeclient", (first,), "gw0")
publish("fakeclient", (), "gw2")
assert sorted(path.name for path in shard_dir(results).iterdir()) == ["gw0.json", "gw1.json"]
assert not results.exists()
publish("fakeclient", (), None)
assert read_rows(results) == (first, second)
def test_clear_shards_drops_a_previous_runs_rows(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
results: Final = tmp_path / "fakeclient.json"
monkeypatch.setenv(RESULTS_ENV, str(results))
publish("fakeclient", (_row("stale", Passed()),), "gw0")
clear_shards("fakeclient")
publish("fakeclient", (_row("fresh", Passed()),), None)
assert [row.model for row in read_rows(results)] == ["fresh"]
def test_a_run_with_no_cells_leaves_the_results_file_alone(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
results: Final = tmp_path / "fakeclient.json"
monkeypatch.setenv(RESULTS_ENV, str(results))
_ = results.write_text("[]")
publish("fakeclient", (), None)
assert results.read_text() == "[]"

View file

@ -0,0 +1,138 @@
from __future__ import annotations
import json
from datetime import datetime, timezone
from pathlib import Path
from typing import Final
import pytest
from client_apps.compat_results import Failed, NotApplicable, NotTested, Outcome, Passed, ResultRow, write_rows
from client_apps.matrix_builder import (
REGRESSION_EXIT_CODE,
REGRESSIONS,
Manifest,
ManifestFeature,
Matrix,
Regression,
aggregate,
build_matrix,
find_regressions,
load_manifest,
main,
pinned_client_version,
)
MANIFEST: Final = Manifest(
schema_version="1",
client="codex",
package="@openai/codex",
providers=("openai", "vertex_ai"),
features=(ManifestFeature(id="vision", name="Vision"), ManifestFeature(id="web_search", name="Web search")),
)
GENERATED_AT: Final = datetime(2026, 1, 2, tzinfo=timezone.utc)
def _row(model: str | None, outcome: Outcome, feature_id: str = "vision", provider: str = "openai") -> ResultRow:
return ResultRow(feature_id=feature_id, provider=provider, model=model, outcome=outcome)
def _matrix(rows: tuple[ResultRow, ...]) -> Matrix:
return build_matrix(MANIFEST, rows, litellm_version="1.0.0", client_version="0.1.0", generated_at=GENERATED_AT)
@pytest.mark.parametrize(
("rows", "expected"),
(
((_row("a", Passed()), _row("b", Passed())), Passed()),
((_row("a", Passed()), _row("b", Failed(error="no image"))), Failed(error="b: no image")),
(
(_row("a", Failed(error="x")), _row(None, Failed(error="setup broke"))),
Failed(error="a: x; setup broke"),
),
((_row("a", Passed()), _row("b", NotTested())), Passed()),
((_row(None, NotApplicable(reason="no apply_patch")),), NotApplicable(reason="no apply_patch")),
((), NotTested()),
),
)
def test_a_cell_is_green_only_when_no_model_failed(rows: tuple[ResultRow, ...], expected: Outcome) -> None:
assert aggregate(rows) == expected
def test_build_matrix_fills_every_manifest_cell_and_drops_unknown_rows() -> None:
matrix: Final = _matrix(
(
_row("a", Passed()),
_row("a", Failed(error="404"), feature_id="web_search", provider="vertex_ai"),
_row("a", Passed(), feature_id="retired_feature"),
_row("a", Passed(), provider="retired_provider"),
)
)
assert [(feature.id, dict(feature.providers)) for feature in matrix.features] == [
("vision", {"openai": Passed(), "vertex_ai": NotTested()}),
("web_search", {"openai": NotTested(), "vertex_ai": Failed(error="a: 404")}),
]
assert matrix.generated_at == GENERATED_AT.isoformat()
def test_find_regressions_flags_only_green_to_red() -> None:
old: Final = _matrix(
(
_row("a", Passed()),
_row("a", Failed(error="old"), provider="vertex_ai"),
_row("a", Passed(), feature_id="web_search"),
)
)
new: Final = _matrix(
(
_row("a", Failed(error="429")),
_row("a", Failed(error="still red"), provider="vertex_ai"),
_row("a", Failed(error="new cell"), feature_id="web_search", provider="vertex_ai"),
_row("a", Passed(), feature_id="web_search"),
)
)
assert find_regressions(old, new) == (
Regression(feature_id="vision", feature_name="Vision", provider="openai", error="a: 429"),
)
def test_matrix_round_trips_through_json() -> None:
matrix: Final = _matrix((_row("a", Passed()), _row(None, NotApplicable(reason="n/a"), provider="vertex_ai")))
assert Matrix.model_validate_json(matrix.model_dump_json()) == matrix
def test_cli_build_then_regressions_exit_codes(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
manifest: Final = tmp_path / "manifest.yaml"
_ = manifest.write_text(json.dumps(MANIFEST.model_dump(mode="json")))
green_rows: Final = tmp_path / "green.json"
red_rows: Final = tmp_path / "red.json"
write_rows(green_rows, (_row("a", Passed()),))
write_rows(red_rows, (_row("a", Failed(error="400 from proxy")),))
green: Final = tmp_path / "green-matrix.json"
red: Final = tmp_path / "red-matrix.json"
build: Final = ("build", "--manifest", str(manifest), "--litellm-version", "9.9.9")
assert main((*build, "--results", str(green_rows), "--out", str(green))) == 0
assert main((*build, "--results", str(red_rows), "--out", str(red))) == 0
built: Final = Matrix.model_validate_json(red.read_bytes())
assert (built.litellm_version, built.client_version) == ("9.9.9", pinned_client_version("@openai/codex"))
_ = capsys.readouterr()
assert main(("regressions", "--old", str(green), "--new", str(red))) == REGRESSION_EXIT_CODE
regressions: Final = REGRESSIONS.validate_json(capsys.readouterr().out)
assert [regression.error for regression in regressions] == ["a: 400 from proxy"]
assert main(("regressions", "--old", str(red), "--new", str(green))) == 0
assert main(("regressions", "--old", str(tmp_path / "missing.json"), "--new", str(red))) == 0
def test_shipped_manifests_validate_and_pin_their_client() -> None:
manifests: Final = tuple(Path(__file__).resolve().parents[2].glob("e2e/client_apps/*/manifest.yaml"))
assert manifests
for path in manifests:
manifest = load_manifest(path)
assert len({feature.id for feature in manifest.features}) == len(manifest.features)
assert pinned_client_version(manifest.package)