litellm/tests/e2e/codex/harness.py
mateo-berri fa992071f3 test(e2e): build a Codex compatibility table from tests/e2e/codex and add Vertex AI and five rows
Merge main and move the Codex compatibility work onto main's tests/e2e/codex
suite, replacing the parallel tests/e2e/client_apps suite and its harness tests.

A makereport hook in codex/compat_results.py appends one JSON line per decided
cell when CODEX_COMPAT_RESULTS_PATH is set, and codex/matrix_builder.py builds a
table in the Claude Code matrix shape and exits 3 when a cell that passed no
longer does. The row id comes from the test file name and the label from the
class name, so a new row needs no other edit.

The suite now pins @openai/codex 0.166.0 in its own package.json and lockfile,
adds Gemini on Vertex AI as a setup, and adds prompt caching, reasoning effort,
vision, structured output and web search rows. Web search carries strict xfails
where Codex or LiteLLM leaves the search tool out, and auto-compaction now also
expects Vertex AI to fail, for the same bridge gap Claude on Bedrock has
2026-11-03 03:10:05 -08:00

125 lines
4.7 KiB
Python

from __future__ import annotations
import subprocess
import tempfile
from collections.abc import Generator, Mapping
from contextlib import contextmanager
from dataclasses import dataclass, field
from itertools import chain
from pathlib import Path
from typing import Final
from cli_harness.lockdown import isolated_home, locked_down_env
from cli_harness.rate_limiter import RateBucket, RateLimiter
from codex.rollout import Compaction, rollout_compactions
from codex.run import CodexRun, codex_run
from e2e_metadata import step
CODEX_CLI: Final = "codex"
EXEC_TIMEOUT_SECONDS: Final = 600.0
_API_KEY_ENV: Final = "LITELLM_API_KEY"
_PROVIDER: Final = "litellm"
_CONFIG_TOML: Final = """\
model_provider = "{provider}"
approval_policy = "never"
sandbox_mode = "workspace-write"
check_for_update_on_startup = false
[analytics]
enabled = false
[model_providers.{provider}]
name = "LiteLLM"
base_url = "{gateway}/v1"
env_key = "{api_key_env}"
wire_api = "responses"
model_catalog_url = "{gateway}/v1/models"
"""
@dataclass(frozen=True, slots=True)
class CodexHarness:
home: Path
api_key: str = field(repr=False)
rate_bucket: RateBucket
limiter: RateLimiter
cli: str = CODEX_CLI
timeout_seconds: float = EXEC_TIMEOUT_SECONDS
@property
def codex_home(self) -> Path:
return self.home / ".codex"
@step('Run Codex on {model} through the proxy with the flags {flags} and the prompt "{prompt}"')
def exec_with_flags(self, prompt: str, *, model: str, cwd: Path, flags: tuple[str, ...]) -> CodexRun:
return self._run(("exec",), (*flags, "--model", model, "--", prompt), cwd)
@step("Create a workspace folder holding the test's files")
def new_workspace(self, files: Mapping[str, str]) -> Path:
workspace: Final = Path(tempfile.mkdtemp(prefix="workspace-", dir=self.home)).resolve()
for name, content in files.items():
_ = (workspace / name).write_text(content)
return workspace
@step('Run Codex on {model} through the proxy with the prompt "{prompt}"')
def exec(self, prompt: str, *, model: str, cwd: Path, compact_after_tokens: int | None = None) -> CodexRun:
return self._run(("exec",), (*_compaction_flags(compact_after_tokens), "--model", model, "--", prompt), cwd)
@step('Resume Codex thread {thread_id} on {model} through the proxy with the prompt "{prompt}"')
def resume(
self, thread_id: str, prompt: str, *, model: str, cwd: Path, compact_after_tokens: int | None = None
) -> CodexRun:
return self._run(
("exec", "resume"),
(*_compaction_flags(compact_after_tokens), "--model", model, "--", thread_id, prompt),
cwd,
)
def compactions(self, thread_id: str) -> tuple[Compaction, ...]:
rollouts: Final = sorted(self.codex_home.glob(f"sessions/**/rollout-*-{thread_id}.jsonl"))
return tuple(chain.from_iterable(rollout_compactions(rollout.read_text()) for rollout in rollouts))
def _run(self, subcommand: tuple[str, ...], args: tuple[str, ...], cwd: Path) -> CodexRun:
_ = self.limiter.acquire(self.rate_bucket)
completed: Final = subprocess.run(
(self.cli, *subcommand, "--json", "--skip-git-repo-check", *args),
cwd=cwd,
env=locked_down_env(
home=self.home, overrides={"CODEX_HOME": str(self.codex_home), _API_KEY_ENV: self.api_key}
),
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=self.timeout_seconds,
check=False,
)
return codex_run(exit_code=completed.returncode, stdout=completed.stdout, stderr=completed.stderr)
def _compaction_flags(after_tokens: int | None) -> tuple[str, ...]:
match after_tokens:
case None:
return ()
case int():
return (
"--config",
f'model_providers.{_PROVIDER}.capabilities.remote_compaction="v2"',
"--config",
'model_auto_compact_token_limit_scope="body_after_prefix"',
"--config",
f"model_auto_compact_token_limit={after_tokens}",
)
@step("Give Codex a fresh empty home whose config sends every request to the proxy")
@contextmanager
def codex_harness(
*, gateway: str, api_key: str, rate_bucket: RateBucket, limiter: RateLimiter
) -> Generator[CodexHarness]:
with isolated_home("codex-cli-home-") as home:
harness: Final = CodexHarness(home=home, api_key=api_key, rate_bucket=rate_bucket, limiter=limiter)
harness.codex_home.mkdir()
_ = (harness.codex_home / "config.toml").write_text(
_CONFIG_TOML.format(provider=_PROVIDER, gateway=gateway, api_key_env=_API_KEY_ENV)
)
yield harness