litellm/tests/e2e/codex/harness.py
ryan-crabbe-berri 6254860560
test(e2e): drive the Codex CLI through LiteLLM, command round trip (#49374)
* test(e2e): drive the real Codex CLI through the gateway and add the command round trip row

Adds tests/e2e/codex: a harness that runs codex exec --json with a throwaway HOME and CODEX_HOME, a custom responses provider pointed at the gateway, the model list read from the gateway's /v1/models, the sandbox on and approvals off. Each test registers its setup's deployment via /model/new and calls it with a key scoped to it, both deleted on teardown. The first row checks that a value only a command can print comes back through the gateway on all five setups

* test(e2e): type the Codex self-test event helper as a Mapping

* test(e2e): clean up Codex test keys even when setup fails, and fail runs where Codex exits with an error

* test(e2e): record a step when the Codex harness sets up its home and workspace

The suite rules ask every public harness helper that does something to record a step, so a failed run shows whether it died while setting up Codex or while running it

* Update run.py

wip
2026-11-02 17:32:16 -08:00

86 lines
2.9 KiB
Python

from __future__ import annotations
import subprocess
import tempfile
from collections.abc import Generator, Mapping
from contextlib import contextmanager
from dataclasses import dataclass, field
from pathlib import Path
from typing import Final
from cli_harness.lockdown import isolated_home, locked_down_env
from cli_harness.rate_limiter import RateBucket, RateLimiter
from codex.run import CodexRun, codex_run
from e2e_metadata import step
CODEX_CLI: Final = "codex"
EXEC_TIMEOUT_SECONDS: Final = 600.0
_API_KEY_ENV: Final = "LITELLM_API_KEY"
_CONFIG_TOML: Final = """\
model_provider = "litellm"
approval_policy = "never"
sandbox_mode = "workspace-write"
check_for_update_on_startup = false
[analytics]
enabled = false
[model_providers.litellm]
name = "LiteLLM"
base_url = "{gateway}/v1"
env_key = "{api_key_env}"
wire_api = "responses"
model_catalog_url = "{gateway}/v1/models"
"""
@dataclass(frozen=True, slots=True)
class CodexHarness:
home: Path
api_key: str = field(repr=False)
rate_bucket: RateBucket
limiter: RateLimiter
cli: str = CODEX_CLI
timeout_seconds: float = EXEC_TIMEOUT_SECONDS
@property
def codex_home(self) -> Path:
return self.home / ".codex"
@step("Create a workspace folder holding the test's files")
def new_workspace(self, files: Mapping[str, str]) -> Path:
workspace: Final = Path(tempfile.mkdtemp(prefix="workspace-", dir=self.home))
for name, content in files.items():
_ = (workspace / name).write_text(content)
return workspace
@step('Run Codex on {model} through the proxy with the prompt "{prompt}"')
def exec(self, prompt: str, *, model: str, cwd: Path) -> CodexRun:
_ = self.limiter.acquire(self.rate_bucket)
completed: Final = subprocess.run(
(self.cli, "exec", "--json", "--skip-git-repo-check", "--model", model, "--", prompt),
cwd=cwd,
env=locked_down_env(
home=self.home, overrides={"CODEX_HOME": str(self.codex_home), _API_KEY_ENV: self.api_key}
),
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=self.timeout_seconds,
check=False,
)
return codex_run(exit_code=completed.returncode, stdout=completed.stdout, stderr=completed.stderr)
@step("Give Codex a fresh empty home whose config sends every request to the proxy")
@contextmanager
def codex_harness(
*, gateway: str, api_key: str, rate_bucket: RateBucket, limiter: RateLimiter
) -> Generator[CodexHarness]:
with isolated_home("codex-cli-home-") as home:
harness: Final = CodexHarness(home=home, api_key=api_key, rate_bucket=rate_bucket, limiter=limiter)
harness.codex_home.mkdir()
_ = (harness.codex_home / "config.toml").write_text(
_CONFIG_TOML.format(gateway=gateway, api_key_env=_API_KEY_ENV)
)
yield harness