From 3acdf171ee63b7b8a3cfe6459ce9a2f45110476b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 09:33:07 -0700 Subject: [PATCH] test(e2e): add a Codex compatibility matrix suite Add tests/e2e/client_apps, a compatibility matrix for coding-agent CLIs other than Claude Code, starting with Codex 0.164.0 pinned through npm. Each of the 36 cells drives the real codex exec against the proxy for one feature and provider column, records a pass or fail per model, and the typed matrix builder turns the results into a matrix and flags cells that went from green to red. The harness parts are covered in tests/e2e_harness/client_apps --- tests/e2e/AGENTS.md | 1 + tests/e2e/client_apps/__init__.py | 0 tests/e2e/client_apps/codex/__init__.py | 0 .../codex/basic_messaging/__init__.py | 0 .../codex/basic_messaging/test_azure.py | 27 ++ .../basic_messaging/test_bedrock_mantle.py | 27 ++ .../codex/basic_messaging/test_openai.py | 27 ++ .../codex/basic_messaging/test_vertex_ai.py | 27 ++ tests/e2e/client_apps/codex/cli_driver.py | 210 ++++++++++++ tests/e2e/client_apps/codex/columns.py | 46 +++ tests/e2e/client_apps/codex/conftest.py | 63 ++++ tests/e2e/client_apps/codex/events.py | 186 +++++++++++ tests/e2e/client_apps/codex/features.py | 299 ++++++++++++++++++ .../client_apps/codex/file_edit/__init__.py | 0 .../client_apps/codex/file_edit/test_azure.py | 28 ++ .../codex/file_edit/test_bedrock_mantle.py | 28 ++ .../codex/file_edit/test_openai.py | 28 ++ .../codex/file_edit/test_vertex_ai.py | 28 ++ tests/e2e/client_apps/codex/manifest.yaml | 29 ++ .../client_apps/codex/multi_turn/__init__.py | 0 .../codex/multi_turn/test_azure.py | 27 ++ .../codex/multi_turn/test_bedrock_mantle.py | 27 ++ .../codex/multi_turn/test_openai.py | 27 ++ .../codex/multi_turn/test_vertex_ai.py | 27 ++ .../codex/prompt_caching/__init__.py | 0 .../codex/prompt_caching/test_azure.py | 28 ++ .../prompt_caching/test_bedrock_mantle.py | 28 ++ .../codex/prompt_caching/test_openai.py | 28 ++ .../codex/prompt_caching/test_vertex_ai.py | 28 ++ tests/e2e/client_apps/codex/proxy_models.yaml | 47 +++ .../codex/reasoning_effort/__init__.py | 0 .../codex/reasoning_effort/test_azure.py | 28 ++ .../reasoning_effort/test_bedrock_mantle.py | 28 ++ .../codex/reasoning_effort/test_openai.py | 28 ++ .../codex/reasoning_effort/test_vertex_ai.py | 28 ++ tests/e2e/client_apps/codex/registry.py | 47 +++ .../codex/shell_command/__init__.py | 0 .../codex/shell_command/test_azure.py | 28 ++ .../shell_command/test_bedrock_mantle.py | 28 ++ .../codex/shell_command/test_openai.py | 28 ++ .../codex/shell_command/test_vertex_ai.py | 28 ++ .../codex/structured_output/__init__.py | 0 .../codex/structured_output/test_azure.py | 28 ++ .../structured_output/test_bedrock_mantle.py | 28 ++ .../codex/structured_output/test_openai.py | 28 ++ .../codex/structured_output/test_vertex_ai.py | 28 ++ .../e2e/client_apps/codex/vision/__init__.py | 0 .../client_apps/codex/vision/test_azure.py | 28 ++ .../codex/vision/test_bedrock_mantle.py | 28 ++ .../client_apps/codex/vision/test_openai.py | 28 ++ .../codex/vision/test_vertex_ai.py | 28 ++ .../client_apps/codex/web_search/__init__.py | 0 .../codex/web_search/test_azure.py | 28 ++ .../codex/web_search/test_bedrock_mantle.py | 28 ++ .../codex/web_search/test_openai.py | 28 ++ .../codex/web_search/test_vertex_ai.py | 28 ++ tests/e2e/client_apps/compat_collector.py | 139 ++++++++ tests/e2e/client_apps/compat_results.py | 74 +++++ tests/e2e/client_apps/conftest.py | 59 ++++ tests/e2e/client_apps/matrix_builder.py | 215 +++++++++++++ tests/e2e/client_apps/package-lock.json | 137 ++++++++ tests/e2e/client_apps/package.json | 8 + .../coverage_registry/llm_codex_compat.yaml | 40 +++ tests/e2e/coverage_registry/schema.py | 1 + tests/e2e_harness/AGENTS.md | 4 +- .../codex/test_codex_cli_driver.py | 148 +++++++++ .../client_apps/codex/test_codex_columns.py | 40 +++ .../client_apps/codex/test_codex_events.py | 154 +++++++++ .../codex/transcripts/fallback_metadata.jsonl | 6 + .../codex/transcripts/file_change.jsonl | 7 + .../codex/transcripts/first_turn.jsonl | 4 + .../codex/transcripts/rate_limited.jsonl | 9 + .../codex/transcripts/resumed_turn.jsonl | 4 + .../codex/transcripts/shell_command.jsonl | 6 + .../codex/transcripts/turn_failed.jsonl | 4 + .../codex/transcripts/web_search.jsonl | 7 + .../web_search_never_completed.jsonl | 7 + .../client_apps/test_compat_collector.py | 140 ++++++++ .../client_apps/test_compat_matrix_builder.py | 138 ++++++++ 79 files changed, 3277 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/client_apps/__init__.py create mode 100644 tests/e2e/client_apps/codex/__init__.py create mode 100644 tests/e2e/client_apps/codex/basic_messaging/__init__.py create mode 100644 tests/e2e/client_apps/codex/basic_messaging/test_azure.py create mode 100644 tests/e2e/client_apps/codex/basic_messaging/test_bedrock_mantle.py create mode 100644 tests/e2e/client_apps/codex/basic_messaging/test_openai.py create mode 100644 tests/e2e/client_apps/codex/basic_messaging/test_vertex_ai.py create mode 100644 tests/e2e/client_apps/codex/cli_driver.py create mode 100644 tests/e2e/client_apps/codex/columns.py create mode 100644 tests/e2e/client_apps/codex/conftest.py create mode 100644 tests/e2e/client_apps/codex/events.py create mode 100644 tests/e2e/client_apps/codex/features.py create mode 100644 tests/e2e/client_apps/codex/file_edit/__init__.py create mode 100644 tests/e2e/client_apps/codex/file_edit/test_azure.py create mode 100644 tests/e2e/client_apps/codex/file_edit/test_bedrock_mantle.py create mode 100644 tests/e2e/client_apps/codex/file_edit/test_openai.py create mode 100644 tests/e2e/client_apps/codex/file_edit/test_vertex_ai.py create mode 100644 tests/e2e/client_apps/codex/manifest.yaml create mode 100644 tests/e2e/client_apps/codex/multi_turn/__init__.py create mode 100644 tests/e2e/client_apps/codex/multi_turn/test_azure.py create mode 100644 tests/e2e/client_apps/codex/multi_turn/test_bedrock_mantle.py create mode 100644 tests/e2e/client_apps/codex/multi_turn/test_openai.py create mode 100644 tests/e2e/client_apps/codex/multi_turn/test_vertex_ai.py create mode 100644 tests/e2e/client_apps/codex/prompt_caching/__init__.py create mode 100644 tests/e2e/client_apps/codex/prompt_caching/test_azure.py create mode 100644 tests/e2e/client_apps/codex/prompt_caching/test_bedrock_mantle.py create mode 100644 tests/e2e/client_apps/codex/prompt_caching/test_openai.py create mode 100644 tests/e2e/client_apps/codex/prompt_caching/test_vertex_ai.py create mode 100644 tests/e2e/client_apps/codex/proxy_models.yaml create mode 100644 tests/e2e/client_apps/codex/reasoning_effort/__init__.py create mode 100644 tests/e2e/client_apps/codex/reasoning_effort/test_azure.py create mode 100644 tests/e2e/client_apps/codex/reasoning_effort/test_bedrock_mantle.py create mode 100644 tests/e2e/client_apps/codex/reasoning_effort/test_openai.py create mode 100644 tests/e2e/client_apps/codex/reasoning_effort/test_vertex_ai.py create mode 100644 tests/e2e/client_apps/codex/registry.py create mode 100644 tests/e2e/client_apps/codex/shell_command/__init__.py create mode 100644 tests/e2e/client_apps/codex/shell_command/test_azure.py create mode 100644 tests/e2e/client_apps/codex/shell_command/test_bedrock_mantle.py create mode 100644 tests/e2e/client_apps/codex/shell_command/test_openai.py create mode 100644 tests/e2e/client_apps/codex/shell_command/test_vertex_ai.py create mode 100644 tests/e2e/client_apps/codex/structured_output/__init__.py create mode 100644 tests/e2e/client_apps/codex/structured_output/test_azure.py create mode 100644 tests/e2e/client_apps/codex/structured_output/test_bedrock_mantle.py create mode 100644 tests/e2e/client_apps/codex/structured_output/test_openai.py create mode 100644 tests/e2e/client_apps/codex/structured_output/test_vertex_ai.py create mode 100644 tests/e2e/client_apps/codex/vision/__init__.py create mode 100644 tests/e2e/client_apps/codex/vision/test_azure.py create mode 100644 tests/e2e/client_apps/codex/vision/test_bedrock_mantle.py create mode 100644 tests/e2e/client_apps/codex/vision/test_openai.py create mode 100644 tests/e2e/client_apps/codex/vision/test_vertex_ai.py create mode 100644 tests/e2e/client_apps/codex/web_search/__init__.py create mode 100644 tests/e2e/client_apps/codex/web_search/test_azure.py create mode 100644 tests/e2e/client_apps/codex/web_search/test_bedrock_mantle.py create mode 100644 tests/e2e/client_apps/codex/web_search/test_openai.py create mode 100644 tests/e2e/client_apps/codex/web_search/test_vertex_ai.py create mode 100644 tests/e2e/client_apps/compat_collector.py create mode 100644 tests/e2e/client_apps/compat_results.py create mode 100644 tests/e2e/client_apps/conftest.py create mode 100644 tests/e2e/client_apps/matrix_builder.py create mode 100644 tests/e2e/client_apps/package-lock.json create mode 100644 tests/e2e/client_apps/package.json create mode 100644 tests/e2e/coverage_registry/llm_codex_compat.yaml create mode 100644 tests/e2e_harness/client_apps/codex/test_codex_cli_driver.py create mode 100644 tests/e2e_harness/client_apps/codex/test_codex_columns.py create mode 100644 tests/e2e_harness/client_apps/codex/test_codex_events.py create mode 100644 tests/e2e_harness/client_apps/codex/transcripts/fallback_metadata.jsonl create mode 100644 tests/e2e_harness/client_apps/codex/transcripts/file_change.jsonl create mode 100644 tests/e2e_harness/client_apps/codex/transcripts/first_turn.jsonl create mode 100644 tests/e2e_harness/client_apps/codex/transcripts/rate_limited.jsonl create mode 100644 tests/e2e_harness/client_apps/codex/transcripts/resumed_turn.jsonl create mode 100644 tests/e2e_harness/client_apps/codex/transcripts/shell_command.jsonl create mode 100644 tests/e2e_harness/client_apps/codex/transcripts/turn_failed.jsonl create mode 100644 tests/e2e_harness/client_apps/codex/transcripts/web_search.jsonl create mode 100644 tests/e2e_harness/client_apps/codex/transcripts/web_search_never_completed.jsonl create mode 100644 tests/e2e_harness/client_apps/test_compat_collector.py create mode 100644 tests/e2e_harness/client_apps/test_compat_matrix_builder.py diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index cf1e41a59a6..be7a1240fbc 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -49,6 +49,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `secret_manager/` - the gateway's `key_management_system` against a real secret manager: deployment keys resolved from it (`os.environ/` where the name exists only in the manager) and virtual keys written to and deleted from it. The tests are backend-agnostic and each backend is its own lane, because the setting is global to the proxy: `E2E_SECRET_MANAGER=` opts in and picks the backend from `secret_backends.BACKENDS`, the proxy is booted from `gateway/secret_manager__ci_config.yml` against the live manager, and the tests reach that manager through the backend's `SecretStore` (`secret_store_.py`). A test needing something not every backend does carries `requires_capability(...)` and is deselected on lanes that lack it. `secret_manager/backend.sh up ` runs a backend in Docker and writes the proxy's and the tests' env. Marked `secret_manager`, deselected unless `E2E_SECRET_MANAGER` is set, and kept out of the per-PR selector. Backends today: `hashicorp_vault` and `cyberark` (CyberArk Conjur, which cannot delete, so the delete test is Vault-only) - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke +- `client_apps/` - compatibility matrices for coding-agent CLIs other than Claude Code, one subfolder per client (`codex/` today). A client folder holds `manifest.yaml` (its provider columns and feature rows), `proxy_models.yaml` (the deployments each column registers through `/model/new` on first use and deletes at session end) and one cell per `/test_.py`. A cell drives the real CLI against the proxy with a scoped key for every model in its column and reports one tagged-union outcome per model through the `compat_result` fixture, and the matrix marks the cell green only when every model passed. Cells of one column take turns across xdist workers through a file lock, since a provider's rate limit is shared by the whole column, while different columns run in parallel. The CLIs are pinned in `client_apps/package.json` and its lockfile, the runner image installs them with `npm ci`, and the driver runs `node_modules/.bin/` before falling back to the one on `PATH`. Nothing here imports `claude_code/`. To run it locally, run `npm ci` in `tests/e2e/client_apps`, then `uv run pytest tests/e2e/client_apps -n 8` with `LITELLM_PROXY_URL` and `LITELLM_MASTER_KEY` set. Point pytest at `tests/e2e/client_apps` or below, because `client_apps/conftest.py` is what writes the results to `_COMPAT_RESULTS_PATH` (default `-compat-results.json`, merged across xdist workers). `cd tests/e2e && python -m client_apps.matrix_builder build --manifest client_apps/codex/manifest.yaml --results --out ` turns them into the matrix, and `regressions --old --new ` exits 3 when a green cell turned red. The builder, collector, parser and driver are covered by `tests/e2e_harness/client_apps/` - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream, see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport. The Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`. Its specs follow the dashboard's rule against testing what shadcn and Base UI already guarantee, in `ui/litellm-dashboard/AGENTS.md` ## MCP suite: real Datadog only diff --git a/tests/e2e/client_apps/__init__.py b/tests/e2e/client_apps/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/client_apps/codex/__init__.py b/tests/e2e/client_apps/codex/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/client_apps/codex/basic_messaging/__init__.py b/tests/e2e/client_apps/codex/basic_messaging/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/client_apps/codex/basic_messaging/test_azure.py b/tests/e2e/client_apps/codex/basic_messaging/test_azure.py new file mode 100644 index 00000000000..1b52dbc189b --- /dev/null +++ b/tests/e2e/client_apps/codex/basic_messaging/test_azure.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, basic_messaging, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("azure") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.azure_openai.basic.stream.codex_basic_messaging") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.AZURE,), + models=MODELS, + mode=Mode.STREAM, + ) +) +def test_basic_messaging_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, basic_messaging) diff --git a/tests/e2e/client_apps/codex/basic_messaging/test_bedrock_mantle.py b/tests/e2e/client_apps/codex/basic_messaging/test_bedrock_mantle.py new file mode 100644 index 00000000000..986b86e4455 --- /dev/null +++ b/tests/e2e/client_apps/codex/basic_messaging/test_bedrock_mantle.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, basic_messaging, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("bedrock_mantle") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.bedrock_mantle.basic.stream.codex_basic_messaging") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.BEDROCK_MANTLE,), + models=MODELS, + mode=Mode.STREAM, + ) +) +def test_basic_messaging_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, basic_messaging) diff --git a/tests/e2e/client_apps/codex/basic_messaging/test_openai.py b/tests/e2e/client_apps/codex/basic_messaging/test_openai.py new file mode 100644 index 00000000000..656ca29ac30 --- /dev/null +++ b/tests/e2e/client_apps/codex/basic_messaging/test_openai.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, basic_messaging, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("openai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.openai.basic.stream.codex_basic_messaging") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=MODELS, + mode=Mode.STREAM, + ) +) +def test_basic_messaging_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, basic_messaging) diff --git a/tests/e2e/client_apps/codex/basic_messaging/test_vertex_ai.py b/tests/e2e/client_apps/codex/basic_messaging/test_vertex_ai.py new file mode 100644 index 00000000000..71d95339a9e --- /dev/null +++ b/tests/e2e/client_apps/codex/basic_messaging/test_vertex_ai.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, basic_messaging, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("vertex_ai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.vertex.basic.stream.codex_basic_messaging") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.VERTEX_AI,), + models=MODELS, + mode=Mode.STREAM, + ) +) +def test_basic_messaging_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, basic_messaging) diff --git a/tests/e2e/client_apps/codex/cli_driver.py b/tests/e2e/client_apps/codex/cli_driver.py new file mode 100644 index 00000000000..b7769b1d2d5 --- /dev/null +++ b/tests/e2e/client_apps/codex/cli_driver.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import json +import os +import shutil +import signal +import subprocess +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from enum import StrEnum +from itertools import chain +from pathlib import Path +from types import MappingProxyType +from typing import Final + +from e2e_metadata import step + +from client_apps.codex.events import Transcript, parse_transcript + +CLIENT_APPS_DIR: Final = Path(__file__).resolve().parents[1] +LOCAL_BINARY: Final = CLIENT_APPS_DIR / "node_modules" / ".bin" / "codex" +TIMEOUT_ENV: Final = "LITELLM_COMPAT_CODEX_TIMEOUT_SECONDS" +DEFAULT_TIMEOUT_SECONDS: Final = 300.0 +LITELLM_PROVIDER_KEY_ENV: Final = "LITELLM_API_KEY" +OPENAI_PROVIDER_KEY_ENV: Final = "CODEX_API_KEY" +SANDBOX_MODE: Final = "danger-full-access" +STDERR_TAIL_CHARS: Final = 600 + + +class Wiring(StrEnum): + LITELLM_PROVIDER = "litellm_provider" + OPENAI_BASE_URL = "openai_base_url" + + +@dataclass(frozen=True, slots=True) +class Gateway: + base_url: str + api_key: str = field(repr=False) + + +@dataclass(frozen=True, slots=True) +class CodexSession: + root: Path + + @property + def codex_home(self) -> Path: + return self.root / "codex_home" + + @property + def home(self) -> Path: + return self.root / "home" + + @property + def workdir(self) -> Path: + return self.root / "workdir" + + +def new_session(root: Path) -> CodexSession: + session: Final = CodexSession(root=root) + for directory in (session.codex_home, session.home, session.workdir): + directory.mkdir(parents=True, exist_ok=True) + return session + + +@dataclass(frozen=True, slots=True) +class CodexTurn: + model: str + prompt: str + options: tuple[str, ...] = () + resume_thread: str | None = None + wiring: Wiring = Wiring.LITELLM_PROVIDER + + +@dataclass(frozen=True, slots=True) +class Finished: + exit_code: int + transcript: Transcript + stderr: str + + +@dataclass(frozen=True, slots=True) +class TimedOut: + seconds: float + transcript: Transcript + stderr: str + + +type CodexRun = Finished | TimedOut + + +def resolve_binary(local: Path = LOCAL_BINARY, which: Callable[[str], str | None] = shutil.which) -> Path | None: + if local.is_file(): + return local + on_path: Final = which("codex") + return Path(on_path) if on_path is not None else None + + +def timeout_seconds(environ: Mapping[str, str]) -> float: + return float(environ.get(TIMEOUT_ENV, DEFAULT_TIMEOUT_SECONDS)) + + +def _toml_string(value: str) -> str: + return json.dumps(value) + + +def _overrides(settings: Mapping[str, str]) -> tuple[str, ...]: + return tuple(chain.from_iterable(("-c", f"{key}={_toml_string(value)}") for key, value in settings.items())) + + +def wiring_args(wiring: Wiring, base_url: str) -> tuple[str, ...]: + match wiring: + case Wiring.LITELLM_PROVIDER: + return _overrides( + MappingProxyType( + { + "model_provider": "litellm", + "model_providers.litellm.name": "LiteLLM", + "model_providers.litellm.base_url": f"{base_url}/v1", + "model_providers.litellm.env_key": LITELLM_PROVIDER_KEY_ENV, + "model_providers.litellm.wire_api": "responses", + } + ) + ) + case Wiring.OPENAI_BASE_URL: + return _overrides(MappingProxyType({"openai_base_url": f"{base_url}/v1"})) + + +def _key_env(wiring: Wiring) -> str: + match wiring: + case Wiring.LITELLM_PROVIDER: + return LITELLM_PROVIDER_KEY_ENV + case Wiring.OPENAI_BASE_URL: + return OPENAI_PROVIDER_KEY_ENV + + +def command(binary: Path, gateway: Gateway, session: CodexSession, turn: CodexTurn) -> tuple[str, ...]: + resume: Final = ("resume", turn.resume_thread) if turn.resume_thread is not None else () + return ( + str(binary), + "exec", + "--json", + "--skip-git-repo-check", + "--ignore-user-config", + "-C", + str(session.workdir), + "-m", + turn.model, + "-s", + SANDBOX_MODE, + *_overrides(MappingProxyType({"approval_policy": "never"})), + *wiring_args(turn.wiring, gateway.base_url), + *turn.options, + *resume, + "--", + turn.prompt, + ) + + +def environment(gateway: Gateway, session: CodexSession, wiring: Wiring) -> Mapping[str, str]: + return MappingProxyType( + { + "PATH": os.environ.get("PATH", os.defpath), + "HOME": str(session.home), + "CODEX_HOME": str(session.codex_home), + _key_env(wiring): gateway.api_key, + } + ) + + +def _turn_files(session: CodexSession) -> tuple[Path, Path]: + turn: Final = len(tuple(session.root.glob("turn-*.jsonl"))) + 1 + return session.root / f"turn-{turn}.jsonl", session.root / f"turn-{turn}.stderr.txt" + + +@step("Run codex exec against {turn.model}") +def run_codex(binary: Path, gateway: Gateway, session: CodexSession, turn: CodexTurn, seconds: float) -> CodexRun: + stdout_path, stderr_path = _turn_files(session) + with ( + stdout_path.open("w") as stdout, + stderr_path.open("w") as stderr, + subprocess.Popen( + command(binary, gateway, session, turn), + cwd=session.workdir, + env=environment(gateway, session, turn.wiring), + stdin=subprocess.DEVNULL, + stdout=stdout, + stderr=stderr, + start_new_session=True, + ) as process, + ): + try: + exit_code: Final = process.wait(timeout=seconds) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + _ = process.wait() + return TimedOut( + seconds=seconds, + transcript=parse_transcript(stdout_path.read_text()), + stderr=stderr_path.read_text(), + ) + return Finished( + exit_code=exit_code, transcript=parse_transcript(stdout_path.read_text()), stderr=stderr_path.read_text() + ) + + +def stderr_tail(stderr: str) -> str: + lines: Final = tuple( + line for line in stderr.splitlines() if line.strip() and not line.startswith("Reading additional input") + ) + return "\n".join(lines)[-STDERR_TAIL_CHARS:] diff --git a/tests/e2e/client_apps/codex/columns.py b/tests/e2e/client_apps/codex/columns.py new file mode 100644 index 00000000000..5fbee29defa --- /dev/null +++ b/tests/e2e/client_apps/codex/columns.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import fcntl +import tempfile +from collections.abc import Generator, Mapping +from contextlib import contextmanager +from functools import cache +from pathlib import Path +from typing import Final + +import yaml +from models import LiteLLMParamsBody +from pydantic import BaseModel, ConfigDict, TypeAdapter + +PROXY_MODELS_PATH: Final = Path(__file__).resolve().parent / "proxy_models.yaml" +COLUMN_LOCK_DIR: Final = Path(tempfile.gettempdir()) / "litellm-codex-compat-columns" + + +class Deployment(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + model_name: str + litellm_params: LiteLLMParamsBody + + +COLUMNS: Final = TypeAdapter(Mapping[str, tuple[Deployment, ...]]) + + +@cache +def load_columns(path: Path = PROXY_MODELS_PATH) -> Mapping[str, tuple[Deployment, ...]]: + return COLUMNS.validate_python(yaml.safe_load(path.read_text())) + + +def column_models(provider: str) -> tuple[str, ...]: + return tuple(deployment.model_name for deployment in load_columns()[provider]) + + +@contextmanager +def column_lock(provider: str, lock_dir: Path = COLUMN_LOCK_DIR) -> Generator[None]: + lock_dir.mkdir(parents=True, exist_ok=True) + with (lock_dir / f"{provider}.lock").open("a") as handle: + fcntl.flock(handle, fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(handle, fcntl.LOCK_UN) diff --git a/tests/e2e/client_apps/codex/conftest.py b/tests/e2e/client_apps/codex/conftest.py new file mode 100644 index 00000000000..06382bdc502 --- /dev/null +++ b/tests/e2e/client_apps/codex/conftest.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import os +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import pytest +from e2e_config import PROXY_BASE_URL +from proxy_client import ProxyClient + +from client_apps.codex.cli_driver import Gateway, resolve_binary, timeout_seconds +from client_apps.codex.columns import column_lock, load_columns +from client_apps.codex.features import CodexCell +from client_apps.codex.registry import ColumnRegistry +from client_apps.compat_collector import CompatRecorder + + +@dataclass(frozen=True, slots=True) +class CodexSuiteClient: + proxy: ProxyClient + + +@pytest.fixture(scope="session") +def client(proxy: ProxyClient) -> CodexSuiteClient: + return CodexSuiteClient(proxy=proxy) + + +@pytest.fixture(scope="session") +def codex_binary() -> Path: + binary: Final = resolve_binary() + if binary is None: + pytest.fail("No codex binary: run `npm ci` in tests/e2e/client_apps or put codex on PATH") + return binary + + +@pytest.fixture(scope="session") +def column_registry(proxy: ProxyClient) -> Iterator[ColumnRegistry]: + registry: Final = ColumnRegistry(proxy, load_columns()) + yield registry + registry.release() + + +@pytest.fixture +def codex_cell( + compat_result: CompatRecorder, + column_registry: ColumnRegistry, + codex_binary: Path, + scoped_key: str, + tmp_path: Path, +) -> Iterator[CodexCell]: + provider: Final = compat_result.address.provider + errors: Final = column_registry.ensure(provider) + if errors: + pytest.fail("; ".join(f"could not register {error.model_name}: {error.error}" for error in errors)) + with column_lock(provider): + yield CodexCell( + binary=codex_binary, + gateway=Gateway(base_url=PROXY_BASE_URL, api_key=scoped_key), + seconds=timeout_seconds(os.environ), + sessions_root=tmp_path, + ) diff --git a/tests/e2e/client_apps/codex/events.py b/tests/e2e/client_apps/codex/events.py new file mode 100644 index 00000000000..c25f6320572 --- /dev/null +++ b/tests/e2e/client_apps/codex/events.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Annotated, Final, Literal + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError + + +class _Event(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + +class AgentMessage(_Event): + type: Literal["agent_message"] + text: str + + +class Reasoning(_Event): + type: Literal["reasoning"] + text: str = "" + + +class CommandExecution(_Event): + type: Literal["command_execution"] + command: str + aggregated_output: str = "" + exit_code: int | None = None + status: str + + +class FileUpdate(_Event): + path: str + kind: str + + +class FileChange(_Event): + type: Literal["file_change"] + changes: tuple[FileUpdate, ...] + status: str + + +class WebSearch(_Event): + type: Literal["web_search"] + query: str = "" + + +class McpToolCall(_Event): + type: Literal["mcp_tool_call"] + server: str = "" + tool: str = "" + status: str = "" + + +class TodoList(_Event): + type: Literal["todo_list"] + + +class ItemError(_Event): + type: Literal["error"] + message: str + + +class UnknownItem(_Event): + type: str + + +type Item = Annotated[ + AgentMessage + | Reasoning + | CommandExecution + | FileChange + | WebSearch + | McpToolCall + | TodoList + | ItemError + | UnknownItem, + Field(union_mode="left_to_right"), +] + + +class Usage(_Event): + input_tokens: int = 0 + cached_input_tokens: int = 0 + output_tokens: int = 0 + reasoning_output_tokens: int = 0 + + +class ErrorDetail(_Event): + message: str + + +class ThreadStarted(_Event): + type: Literal["thread.started"] + thread_id: str + + +class TurnStarted(_Event): + type: Literal["turn.started"] + + +class TurnCompleted(_Event): + type: Literal["turn.completed"] + usage: Usage + + +class TurnFailed(_Event): + type: Literal["turn.failed"] + error: ErrorDetail + + +class ItemEvent(_Event): + type: Literal["item.started", "item.updated", "item.completed"] + item: Item + + +class StreamError(_Event): + type: Literal["error"] + message: str + + +class UnknownEvent(_Event): + type: str + + +type CodexEvent = Annotated[ + ThreadStarted | TurnStarted | TurnCompleted | TurnFailed | ItemEvent | StreamError | UnknownEvent, + Field(union_mode="left_to_right"), +] + +EVENT: Final = TypeAdapter[CodexEvent](CodexEvent) + + +@dataclass(frozen=True, slots=True) +class UnparsedLine: + text: str + error: str + + +def parse_line(line: str) -> CodexEvent | UnparsedLine: + try: + return EVENT.validate_json(line) + except ValidationError as error: + return UnparsedLine(text=line, error=str(error)) + + +@dataclass(frozen=True, slots=True) +class Transcript: + events: tuple[CodexEvent, ...] + unparsed: tuple[UnparsedLine, ...] + + @property + def thread_id(self) -> str | None: + return next((event.thread_id for event in self.events if isinstance(event, ThreadStarted)), None) + + @property + def completed_items(self) -> tuple[Item, ...]: + return tuple( + event.item for event in self.events if isinstance(event, ItemEvent) and event.type == "item.completed" + ) + + @property + def final_message(self) -> str | None: + messages: Final = tuple(item.text for item in self.completed_items if isinstance(item, AgentMessage)) + return messages[-1] if messages else None + + @property + def usage(self) -> Usage | None: + turns: Final = tuple(event.usage for event in self.events if isinstance(event, TurnCompleted)) + return turns[-1] if turns else None + + @property + def failure(self) -> str | None: + turn_failures: Final = tuple(event.error.message for event in self.events if isinstance(event, TurnFailed)) + turn_failure: Final = turn_failures[0] if turn_failures else None + if turn_failure is not None or self.usage is not None: + return turn_failure + stream_errors: Final = tuple(event.message for event in self.events if isinstance(event, StreamError)) + return stream_errors[-1] if stream_errors else None + + +def parse_transcript(stdout: str) -> Transcript: + parsed: Final = tuple(parse_line(line) for line in stdout.splitlines() if line.strip()) + return Transcript( + events=tuple(event for event in parsed if not isinstance(event, UnparsedLine)), + unparsed=tuple(event for event in parsed if isinstance(event, UnparsedLine)), + ) diff --git a/tests/e2e/client_apps/codex/features.py b/tests/e2e/client_apps/codex/features.py new file mode 100644 index 00000000000..7815211ece1 --- /dev/null +++ b/tests/e2e/client_apps/codex/features.py @@ -0,0 +1,299 @@ +from __future__ import annotations + +import secrets +import struct +import zlib +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from functools import partial +from itertools import chain +from pathlib import Path +from typing import Final + +import pytest +from e2e_config import unique_marker +from pydantic import BaseModel, ConfigDict, ValidationError + +from client_apps.codex.cli_driver import ( + CodexRun, + CodexSession, + CodexTurn, + Finished, + Gateway, + TimedOut, + Wiring, + new_session, + run_codex, + stderr_tail, +) +from client_apps.codex.events import CommandExecution, FileChange, Transcript, WebSearch +from client_apps.compat_results import Failed, Passed +from client_apps.compat_collector import CompatRecorder + +SWATCHES: Final = (("red", (220, 20, 20)), ("green", (20, 160, 20)), ("blue", (20, 40, 220))) +SWATCH_SIDE: Final = 64 +EXPECTED_DOCS_DOMAIN: Final = "litellm.ai" +OUTPUT_SCHEMA: Final = ( + '{"type": "object", "properties": {"marker": {"type": "string"}, "word_count": {"type": "integer"}}, ' + '"required": ["marker", "word_count"], "additionalProperties": false}' +) + + +@dataclass(frozen=True, slots=True) +class CodexCell: + binary: Path + gateway: Gateway + seconds: float + sessions_root: Path + + def session(self, model: str) -> CodexSession: + return new_session(self.sessions_root / model) + + def run(self, session: CodexSession, turn: CodexTurn) -> CodexRun: + return run_codex(self.binary, self.gateway, session, turn, self.seconds) + + +type Verdict = Passed | Failed +type Feature = Callable[[CodexCell, str], Verdict] +type Check = Callable[[Transcript], str | None] + + +def assert_cell(compat_result: CompatRecorder, cell: CodexCell, models: tuple[str, ...], feature: Feature) -> None: + with ThreadPoolExecutor(max_workers=len(models)) as pool: + verdicts: Final = tuple(pool.map(partial(feature, cell), models)) + for model, verdict in zip(models, verdicts, strict=True): + match verdict: + case Passed(): + compat_result.passed(model) + case Failed(error=error): + compat_result.failed(model, error) + failures: Final = tuple( + f"{model}: {verdict.error}" + for model, verdict in zip(models, verdicts, strict=True) + if isinstance(verdict, Failed) + ) + if failures: + pytest.fail("\n".join(failures), pytrace=False) + + +def run_error(run: CodexRun) -> str | None: + match run: + case TimedOut(seconds=seconds, stderr=stderr): + return f"codex exec timed out after {seconds:.0f}s; stderr: {stderr_tail(stderr)}" + case Finished(transcript=transcript) if transcript.failure is not None: + return f"turn failed: {transcript.failure}" + case Finished(exit_code=exit_code, stderr=stderr) if exit_code != 0: + return f"codex exec exited {exit_code}; stderr: {stderr_tail(stderr)}" + case Finished(transcript=transcript) if transcript.usage is None: + return "codex exec ended without a turn.completed event" + case Finished(): + return None + + +def verdict(run: CodexRun, check: Check) -> Verdict: + error: Final = run_error(run) or check(run.transcript) + return Failed(error=error) if error else Passed() + + +def final_message_has(expected: str) -> Check: + def check(transcript: Transcript) -> str | None: + message: Final = transcript.final_message + if message is not None and expected in message.lower(): + return None + return f"final message {message!r} lacks {expected!r}" + + return check + + +def all_checks(*checks: Check) -> Check: + def check(transcript: Transcript) -> str | None: + return next((error for error in (each(transcript) for each in checks) if error is not None), None) + + return check + + +def basic_messaging(cell: CodexCell, model: str) -> Verdict: + marker: Final = unique_marker() + prompt: Final = f"Reply with exactly this token and nothing else: {marker}" + run: Final = cell.run(cell.session(model), CodexTurn(model, prompt)) + return verdict(run, final_message_has(marker)) + + +def _echoed_by_shell(marker: str) -> Check: + def check(transcript: Transcript) -> str | None: + commands: Final = tuple(item for item in transcript.completed_items if isinstance(item, CommandExecution)) + echoed: Final = any(command.exit_code == 0 and marker in command.aggregated_output for command in commands) + return None if echoed else f"no completed command_execution printed {marker!r}: {commands!r}" + + return check + + +def shell_command(cell: CodexCell, model: str) -> Verdict: + marker: Final = unique_marker() + prompt: Final = f"Run the shell command `echo {marker}` with your shell tool, then reply with its output only." + run: Final = cell.run(cell.session(model), CodexTurn(model, prompt)) + return verdict(run, all_checks(_echoed_by_shell(marker), final_message_has(marker))) + + +def _edited(session: CodexSession, marker: str) -> Check: + target: Final = session.workdir / "notes.txt" + + def check(transcript: Transcript) -> str | None: + changes: Final = tuple(item for item in transcript.completed_items if isinstance(item, FileChange)) + updates: Final = chain.from_iterable(item.changes for item in changes) + touched: Final = tuple(Path(update.path).name for update in updates) + if target.name not in touched: + return f"no completed file_change item touched {target.name}: {changes!r}" + content: Final = target.read_text() if target.is_file() else None + return None if content is not None and marker in content else f"{target.name} holds {content!r}, not {marker!r}" + + return check + + +def file_edit(cell: CodexCell, model: str) -> Verdict: + marker: Final = unique_marker() + session: Final = cell.session(model) + prompt: Final = ( + f"Create a file named notes.txt in the current directory whose only line is {marker}. " + "Use your apply_patch tool to create it, not the shell." + ) + return verdict(cell.run(session, CodexTurn(model, prompt)), _edited(session, marker)) + + +def _resumed(cell: CodexCell, model: str, first_prompt: str, second_prompt: str) -> tuple[CodexRun, CodexRun | str]: + session: Final = cell.session(model) + first: Final = cell.run(session, CodexTurn(model, first_prompt)) + thread_id: Final = first.transcript.thread_id + first_error: Final = run_error(first) + if first_error is not None or thread_id is None: + return first, f"first turn: {first_error or 'no thread.started event'}" + return first, cell.run(session, CodexTurn(model, second_prompt, resume_thread=thread_id)) + + +def _same_thread(thread_id: str | None) -> Check: + def check(transcript: Transcript) -> str | None: + if transcript.thread_id == thread_id: + return None + return f"resumed thread {transcript.thread_id!r} != {thread_id!r}" + + return check + + +def multi_turn(cell: CodexCell, model: str) -> Verdict: + marker: Final = unique_marker() + first, second = _resumed( + cell, + model, + f"Remember this token for later: {marker}. Reply with OK only.", + "What token did I ask you to remember? Reply with the token only.", + ) + if isinstance(second, str): + return Failed(error=second) + return verdict(second, all_checks(_same_thread(first.transcript.thread_id), final_message_has(marker))) + + +def _cache_read(transcript: Transcript) -> str | None: + cached: Final = transcript.usage.cached_input_tokens if transcript.usage is not None else 0 + return None if cached > 0 else f"second turn reported {cached} cached input tokens: {transcript.usage!r}" + + +def prompt_caching(cell: CodexCell, model: str) -> Verdict: + first_marker: Final = unique_marker() + second_marker: Final = unique_marker() + _, second = _resumed( + cell, + model, + f"Reply with exactly this token and nothing else: {first_marker}", + f"Now reply with exactly this token and nothing else: {second_marker}", + ) + if isinstance(second, str): + return Failed(error=second) + return verdict(second, _cache_read) + + +def _reasoned(transcript: Transcript) -> str | None: + tokens: Final = transcript.usage.reasoning_output_tokens if transcript.usage is not None else 0 + return None if tokens > 0 else f"turn reported {tokens} reasoning output tokens: {transcript.usage!r}" + + +def reasoning_effort(cell: CodexCell, model: str) -> Verdict: + marker: Final = unique_marker() + prompt: Final = ( + f"How many prime numbers are there between 1 and 60? Reply with the number only. Request id {marker}." + ) + options: Final = ("-c", 'model_reasoning_effort="high"') + return verdict(cell.run(cell.session(model), CodexTurn(model, prompt, options=options)), _reasoned) + + +def solid_png(rgb: tuple[int, int, int], side: int = SWATCH_SIDE) -> bytes: + row: Final = b"\x00" + bytes(rgb) * side + return b"".join( + ( + b"\x89PNG\r\n\x1a\n", + _png_chunk(b"IHDR", struct.pack(">IIBBBBB", side, side, 8, 2, 0, 0, 0)), + _png_chunk(b"IDAT", zlib.compress(row * side)), + _png_chunk(b"IEND", b""), + ) + ) + + +def _png_chunk(kind: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + +def vision(cell: CodexCell, model: str) -> Verdict: + marker: Final = unique_marker() + color, rgb = secrets.choice(SWATCHES) + session: Final = cell.session(model) + image: Final = session.root / "swatch.png" + _ = image.write_bytes(solid_png(rgb)) + prompt: Final = f"What single color fills the attached image? Reply with one lowercase word. Request id {marker}." + run: Final = cell.run(session, CodexTurn(model, prompt, options=("-i", str(image)))) + return verdict(run, final_message_has(color)) + + +class StructuredAnswer(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", strict=True) + + marker: str + word_count: int + + +def _structured(marker: str) -> Check: + def check(transcript: Transcript) -> str | None: + try: + answer: Final = StructuredAnswer.model_validate_json(transcript.final_message or "") + except ValidationError as error: + return f"final message {transcript.final_message!r} does not match the output schema: {error}" + expected: Final = StructuredAnswer(marker=marker, word_count=4) + return None if answer == expected else f"structured answer {answer!r} != {expected!r}" + + return check + + +def structured_output(cell: CodexCell, model: str) -> Verdict: + marker: Final = unique_marker() + session: Final = cell.session(model) + schema: Final = session.root / "schema.json" + _ = schema.write_text(OUTPUT_SCHEMA) + prompt: Final = ( + f"Put the token {marker} in marker and the number of words in 'the quick brown fox' in word_count." + ) + run: Final = cell.run(session, CodexTurn(model, prompt, options=("--output-schema", str(schema)))) + return verdict(run, _structured(marker)) + + +def _searched(transcript: Transcript) -> str | None: + searches: Final = tuple(item for item in transcript.completed_items if isinstance(item, WebSearch)) + return None if searches else "no completed web_search item" + + +def web_search(cell: CodexCell, model: str) -> Verdict: + marker: Final = unique_marker() + prompt: Final = ( + "Use your web search tool, not the shell, to find the domain of LiteLLM's documentation site. " + f"Reply with the domain only. Request id {marker}." + ) + turn: Final = CodexTurn(model, prompt, options=("-c", 'web_search="live"'), wiring=Wiring.OPENAI_BASE_URL) + return verdict(cell.run(cell.session(model), turn), all_checks(_searched, final_message_has(EXPECTED_DOCS_DOMAIN))) diff --git a/tests/e2e/client_apps/codex/file_edit/__init__.py b/tests/e2e/client_apps/codex/file_edit/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/client_apps/codex/file_edit/test_azure.py b/tests/e2e/client_apps/codex/file_edit/test_azure.py new file mode 100644 index 00000000000..e55697ddbc0 --- /dev/null +++ b/tests/e2e/client_apps/codex/file_edit/test_azure.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, file_edit, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("azure") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.azure_openai.tool_use.stream.codex_file_edit") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.AZURE,), + models=MODELS, + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) +def test_file_edit_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, file_edit) diff --git a/tests/e2e/client_apps/codex/file_edit/test_bedrock_mantle.py b/tests/e2e/client_apps/codex/file_edit/test_bedrock_mantle.py new file mode 100644 index 00000000000..22478a2cd9f --- /dev/null +++ b/tests/e2e/client_apps/codex/file_edit/test_bedrock_mantle.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, file_edit, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("bedrock_mantle") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.bedrock_mantle.tool_use.stream.codex_file_edit") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.BEDROCK_MANTLE,), + models=MODELS, + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) +def test_file_edit_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, file_edit) diff --git a/tests/e2e/client_apps/codex/file_edit/test_openai.py b/tests/e2e/client_apps/codex/file_edit/test_openai.py new file mode 100644 index 00000000000..0a2c2e6f0b8 --- /dev/null +++ b/tests/e2e/client_apps/codex/file_edit/test_openai.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, file_edit, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("openai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.openai.tool_use.stream.codex_file_edit") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=MODELS, + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) +def test_file_edit_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, file_edit) diff --git a/tests/e2e/client_apps/codex/file_edit/test_vertex_ai.py b/tests/e2e/client_apps/codex/file_edit/test_vertex_ai.py new file mode 100644 index 00000000000..9b75dc9ca76 --- /dev/null +++ b/tests/e2e/client_apps/codex/file_edit/test_vertex_ai.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, file_edit, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("vertex_ai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.vertex.tool_use.stream.codex_file_edit") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.VERTEX_AI,), + models=MODELS, + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) +def test_file_edit_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, file_edit) diff --git a/tests/e2e/client_apps/codex/manifest.yaml b/tests/e2e/client_apps/codex/manifest.yaml new file mode 100644 index 00000000000..626356c16d7 --- /dev/null +++ b/tests/e2e/client_apps/codex/manifest.yaml @@ -0,0 +1,29 @@ +schema_version: "1" +client: codex +package: "@openai/codex" + +providers: + - openai + - azure + - bedrock_mantle + - vertex_ai + +features: + - id: basic_messaging + name: Basic messaging + - id: shell_command + name: Shell command + - id: file_edit + name: File edit + - id: multi_turn + name: Multi-turn (resumed thread) + - id: prompt_caching + name: Prompt caching + - id: reasoning_effort + name: Reasoning effort + - id: vision + name: Vision (image input) + - id: structured_output + name: Structured output + - id: web_search + name: Web search diff --git a/tests/e2e/client_apps/codex/multi_turn/__init__.py b/tests/e2e/client_apps/codex/multi_turn/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/client_apps/codex/multi_turn/test_azure.py b/tests/e2e/client_apps/codex/multi_turn/test_azure.py new file mode 100644 index 00000000000..25f44c91698 --- /dev/null +++ b/tests/e2e/client_apps/codex/multi_turn/test_azure.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, multi_turn, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("azure") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.azure_openai.multi_turn.stream.codex_multi_turn") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.AZURE,), + models=MODELS, + mode=Mode.STREAM, + ) +) +def test_multi_turn_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, multi_turn) diff --git a/tests/e2e/client_apps/codex/multi_turn/test_bedrock_mantle.py b/tests/e2e/client_apps/codex/multi_turn/test_bedrock_mantle.py new file mode 100644 index 00000000000..826ed128f83 --- /dev/null +++ b/tests/e2e/client_apps/codex/multi_turn/test_bedrock_mantle.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, multi_turn, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("bedrock_mantle") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.bedrock_mantle.multi_turn.stream.codex_multi_turn") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.BEDROCK_MANTLE,), + models=MODELS, + mode=Mode.STREAM, + ) +) +def test_multi_turn_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, multi_turn) diff --git a/tests/e2e/client_apps/codex/multi_turn/test_openai.py b/tests/e2e/client_apps/codex/multi_turn/test_openai.py new file mode 100644 index 00000000000..4cd991c55f6 --- /dev/null +++ b/tests/e2e/client_apps/codex/multi_turn/test_openai.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, multi_turn, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("openai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.openai.multi_turn.stream.codex_multi_turn") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=MODELS, + mode=Mode.STREAM, + ) +) +def test_multi_turn_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, multi_turn) diff --git a/tests/e2e/client_apps/codex/multi_turn/test_vertex_ai.py b/tests/e2e/client_apps/codex/multi_turn/test_vertex_ai.py new file mode 100644 index 00000000000..5b71a5a5032 --- /dev/null +++ b/tests/e2e/client_apps/codex/multi_turn/test_vertex_ai.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, multi_turn, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("vertex_ai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.vertex.multi_turn.stream.codex_multi_turn") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.VERTEX_AI,), + models=MODELS, + mode=Mode.STREAM, + ) +) +def test_multi_turn_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, multi_turn) diff --git a/tests/e2e/client_apps/codex/prompt_caching/__init__.py b/tests/e2e/client_apps/codex/prompt_caching/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/client_apps/codex/prompt_caching/test_azure.py b/tests/e2e/client_apps/codex/prompt_caching/test_azure.py new file mode 100644 index 00000000000..0b23d2cc754 --- /dev/null +++ b/tests/e2e/client_apps/codex/prompt_caching/test_azure.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, prompt_caching, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("azure") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.azure_openai.prompt_cache_5m.stream.codex_prompt_caching") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.AZURE,), + models=MODELS, + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.STREAM, + ) +) +def test_prompt_caching_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, prompt_caching) diff --git a/tests/e2e/client_apps/codex/prompt_caching/test_bedrock_mantle.py b/tests/e2e/client_apps/codex/prompt_caching/test_bedrock_mantle.py new file mode 100644 index 00000000000..1d89bec38e9 --- /dev/null +++ b/tests/e2e/client_apps/codex/prompt_caching/test_bedrock_mantle.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, prompt_caching, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("bedrock_mantle") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.bedrock_mantle.prompt_cache_5m.stream.codex_prompt_caching") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.BEDROCK_MANTLE,), + models=MODELS, + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.STREAM, + ) +) +def test_prompt_caching_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, prompt_caching) diff --git a/tests/e2e/client_apps/codex/prompt_caching/test_openai.py b/tests/e2e/client_apps/codex/prompt_caching/test_openai.py new file mode 100644 index 00000000000..812a001c47c --- /dev/null +++ b/tests/e2e/client_apps/codex/prompt_caching/test_openai.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, prompt_caching, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("openai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.openai.prompt_cache_5m.stream.codex_prompt_caching") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=MODELS, + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.STREAM, + ) +) +def test_prompt_caching_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, prompt_caching) diff --git a/tests/e2e/client_apps/codex/prompt_caching/test_vertex_ai.py b/tests/e2e/client_apps/codex/prompt_caching/test_vertex_ai.py new file mode 100644 index 00000000000..a59ca4ad942 --- /dev/null +++ b/tests/e2e/client_apps/codex/prompt_caching/test_vertex_ai.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, prompt_caching, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("vertex_ai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.vertex.prompt_cache_5m.stream.codex_prompt_caching") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.VERTEX_AI,), + models=MODELS, + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.STREAM, + ) +) +def test_prompt_caching_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, prompt_caching) diff --git a/tests/e2e/client_apps/codex/proxy_models.yaml b/tests/e2e/client_apps/codex/proxy_models.yaml new file mode 100644 index 00000000000..da82f83ef7a --- /dev/null +++ b/tests/e2e/client_apps/codex/proxy_models.yaml @@ -0,0 +1,47 @@ +openai: + - model_name: gpt-6.1-sol-codex-compat-openai + litellm_params: + model: openai/gpt-6.1-sol + api_key: os.environ/OPENAI_API_KEY + - model_name: gpt-6-luna-codex-compat-openai + litellm_params: + model: openai/gpt-6-luna + api_key: os.environ/OPENAI_API_KEY + +azure: + - model_name: gpt-6.1-sol-codex-compat-azure + litellm_params: + model: azure/gpt-6.1-sol + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + api_version: v1 + - model_name: gpt-6-luna-codex-compat-azure + litellm_params: + model: azure/gpt-6-luna + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + api_version: v1 + +bedrock_mantle: + - model_name: gpt-6.1-sol-codex-compat-bedrock-mantle + litellm_params: + model: bedrock_mantle/openai.gpt-6.1-sol + aws_region_name: us-east-1 + - model_name: gpt-6-luna-codex-compat-bedrock-mantle + litellm_params: + model: bedrock_mantle/openai.gpt-6-luna + aws_region_name: us-east-1 + +vertex_ai: + - model_name: gemini-4-argon-codex-compat-vertex-ai + litellm_params: + model: vertex_ai/gemini-4-argon + vertex_project: os.environ/VERTEXAI_PROJECT + vertex_location: global + vertex_credentials: os.environ/VERTEXAI_CREDENTIALS + - model_name: gemini-3.8-flash-codex-compat-vertex-ai + litellm_params: + model: vertex_ai/gemini-3.8-flash + vertex_project: os.environ/VERTEXAI_PROJECT + vertex_location: global + vertex_credentials: os.environ/VERTEXAI_CREDENTIALS diff --git a/tests/e2e/client_apps/codex/reasoning_effort/__init__.py b/tests/e2e/client_apps/codex/reasoning_effort/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/client_apps/codex/reasoning_effort/test_azure.py b/tests/e2e/client_apps/codex/reasoning_effort/test_azure.py new file mode 100644 index 00000000000..54dc7f0342e --- /dev/null +++ b/tests/e2e/client_apps/codex/reasoning_effort/test_azure.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, reasoning_effort, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("azure") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.azure_openai.thinking.stream.codex_reasoning_effort") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.AZURE,), + models=MODELS, + capabilities=(Capability.REASONING,), + mode=Mode.STREAM, + ) +) +def test_reasoning_effort_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, reasoning_effort) diff --git a/tests/e2e/client_apps/codex/reasoning_effort/test_bedrock_mantle.py b/tests/e2e/client_apps/codex/reasoning_effort/test_bedrock_mantle.py new file mode 100644 index 00000000000..e3b842152dd --- /dev/null +++ b/tests/e2e/client_apps/codex/reasoning_effort/test_bedrock_mantle.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, reasoning_effort, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("bedrock_mantle") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.bedrock_mantle.thinking.stream.codex_reasoning_effort") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.BEDROCK_MANTLE,), + models=MODELS, + capabilities=(Capability.REASONING,), + mode=Mode.STREAM, + ) +) +def test_reasoning_effort_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, reasoning_effort) diff --git a/tests/e2e/client_apps/codex/reasoning_effort/test_openai.py b/tests/e2e/client_apps/codex/reasoning_effort/test_openai.py new file mode 100644 index 00000000000..8d9033c906f --- /dev/null +++ b/tests/e2e/client_apps/codex/reasoning_effort/test_openai.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, reasoning_effort, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("openai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.openai.thinking.stream.codex_reasoning_effort") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=MODELS, + capabilities=(Capability.REASONING,), + mode=Mode.STREAM, + ) +) +def test_reasoning_effort_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, reasoning_effort) diff --git a/tests/e2e/client_apps/codex/reasoning_effort/test_vertex_ai.py b/tests/e2e/client_apps/codex/reasoning_effort/test_vertex_ai.py new file mode 100644 index 00000000000..c125965ba3e --- /dev/null +++ b/tests/e2e/client_apps/codex/reasoning_effort/test_vertex_ai.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, reasoning_effort, assert_cell +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("vertex_ai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.vertex.thinking.stream.codex_reasoning_effort") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.VERTEX_AI,), + models=MODELS, + capabilities=(Capability.REASONING,), + mode=Mode.STREAM, + ) +) +def test_reasoning_effort_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, reasoning_effort) diff --git a/tests/e2e/client_apps/codex/registry.py b/tests/e2e/client_apps/codex/registry.py new file mode 100644 index 00000000000..164a8c2ea7b --- /dev/null +++ b/tests/e2e/client_apps/codex/registry.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +from proxy_client import ProxyClient + +from client_apps.codex.columns import Deployment + + +@dataclass(frozen=True, slots=True) +class RegistrationError: + model_name: str + error: str + + +def _register(proxy: ProxyClient, deployment: Deployment) -> str | RegistrationError: + try: + return proxy.create_model(deployment.model_name, deployment.litellm_params, provider_live=True) + except AssertionError as error: + return RegistrationError(model_name=deployment.model_name, error=str(error)) + + +class ColumnRegistry: + __slots__ = ("_columns", "_proxy", "_registered") + + def __init__(self, proxy: ProxyClient, columns: Mapping[str, tuple[Deployment, ...]]) -> None: + self._proxy: Final = proxy + self._columns: Final = columns + self._registered: Mapping[str, tuple[str | RegistrationError, ...]] = MappingProxyType({}) + + def ensure(self, provider: str) -> tuple[RegistrationError, ...]: + if provider not in self._registered: + self._registered = MappingProxyType( + { + **self._registered, + provider: tuple(_register(self._proxy, deployment) for deployment in self._columns[provider]), + } + ) + return tuple(result for result in self._registered[provider] if isinstance(result, RegistrationError)) + + def release(self) -> None: + for results in self._registered.values(): + for model_id in (result for result in results if isinstance(result, str)): + self._proxy.delete_model(model_id) diff --git a/tests/e2e/client_apps/codex/shell_command/__init__.py b/tests/e2e/client_apps/codex/shell_command/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/client_apps/codex/shell_command/test_azure.py b/tests/e2e/client_apps/codex/shell_command/test_azure.py new file mode 100644 index 00000000000..906a1ac771e --- /dev/null +++ b/tests/e2e/client_apps/codex/shell_command/test_azure.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, assert_cell, shell_command +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("azure") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.azure_openai.tool_use.stream.codex_shell_command") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.AZURE,), + models=MODELS, + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) +def test_shell_command_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, shell_command) diff --git a/tests/e2e/client_apps/codex/shell_command/test_bedrock_mantle.py b/tests/e2e/client_apps/codex/shell_command/test_bedrock_mantle.py new file mode 100644 index 00000000000..00dd5d2c6f6 --- /dev/null +++ b/tests/e2e/client_apps/codex/shell_command/test_bedrock_mantle.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, assert_cell, shell_command +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("bedrock_mantle") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.bedrock_mantle.tool_use.stream.codex_shell_command") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.BEDROCK_MANTLE,), + models=MODELS, + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) +def test_shell_command_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, shell_command) diff --git a/tests/e2e/client_apps/codex/shell_command/test_openai.py b/tests/e2e/client_apps/codex/shell_command/test_openai.py new file mode 100644 index 00000000000..72d75a0d480 --- /dev/null +++ b/tests/e2e/client_apps/codex/shell_command/test_openai.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, assert_cell, shell_command +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("openai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.openai.tool_use.stream.codex_shell_command") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=MODELS, + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) +def test_shell_command_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, shell_command) diff --git a/tests/e2e/client_apps/codex/shell_command/test_vertex_ai.py b/tests/e2e/client_apps/codex/shell_command/test_vertex_ai.py new file mode 100644 index 00000000000..a04fd9eeed9 --- /dev/null +++ b/tests/e2e/client_apps/codex/shell_command/test_vertex_ai.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, assert_cell, shell_command +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("vertex_ai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.vertex.tool_use.stream.codex_shell_command") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.VERTEX_AI,), + models=MODELS, + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.STREAM, + ) +) +def test_shell_command_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, shell_command) diff --git a/tests/e2e/client_apps/codex/structured_output/__init__.py b/tests/e2e/client_apps/codex/structured_output/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/client_apps/codex/structured_output/test_azure.py b/tests/e2e/client_apps/codex/structured_output/test_azure.py new file mode 100644 index 00000000000..7e10818ac1c --- /dev/null +++ b/tests/e2e/client_apps/codex/structured_output/test_azure.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, assert_cell, structured_output +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("azure") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.azure_openai.structured_output.stream.codex_structured_output") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.AZURE,), + models=MODELS, + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.STREAM, + ) +) +def test_structured_output_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, structured_output) diff --git a/tests/e2e/client_apps/codex/structured_output/test_bedrock_mantle.py b/tests/e2e/client_apps/codex/structured_output/test_bedrock_mantle.py new file mode 100644 index 00000000000..3d0d3935d39 --- /dev/null +++ b/tests/e2e/client_apps/codex/structured_output/test_bedrock_mantle.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, assert_cell, structured_output +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("bedrock_mantle") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.bedrock_mantle.structured_output.stream.codex_structured_output") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.BEDROCK_MANTLE,), + models=MODELS, + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.STREAM, + ) +) +def test_structured_output_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, structured_output) diff --git a/tests/e2e/client_apps/codex/structured_output/test_openai.py b/tests/e2e/client_apps/codex/structured_output/test_openai.py new file mode 100644 index 00000000000..9b0c1c55464 --- /dev/null +++ b/tests/e2e/client_apps/codex/structured_output/test_openai.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, assert_cell, structured_output +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("openai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.openai.structured_output.stream.codex_structured_output") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=MODELS, + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.STREAM, + ) +) +def test_structured_output_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, structured_output) diff --git a/tests/e2e/client_apps/codex/structured_output/test_vertex_ai.py b/tests/e2e/client_apps/codex/structured_output/test_vertex_ai.py new file mode 100644 index 00000000000..4ef159ee426 --- /dev/null +++ b/tests/e2e/client_apps/codex/structured_output/test_vertex_ai.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, assert_cell, structured_output +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("vertex_ai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.vertex.structured_output.stream.codex_structured_output") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.VERTEX_AI,), + models=MODELS, + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.STREAM, + ) +) +def test_structured_output_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, structured_output) diff --git a/tests/e2e/client_apps/codex/vision/__init__.py b/tests/e2e/client_apps/codex/vision/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/client_apps/codex/vision/test_azure.py b/tests/e2e/client_apps/codex/vision/test_azure.py new file mode 100644 index 00000000000..5005f8d5f8f --- /dev/null +++ b/tests/e2e/client_apps/codex/vision/test_azure.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, assert_cell, vision +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("azure") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.azure_openai.vision.stream.codex_vision") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.AZURE,), + models=MODELS, + capabilities=(Capability.VISION,), + mode=Mode.STREAM, + ) +) +def test_vision_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, vision) diff --git a/tests/e2e/client_apps/codex/vision/test_bedrock_mantle.py b/tests/e2e/client_apps/codex/vision/test_bedrock_mantle.py new file mode 100644 index 00000000000..556cad6c666 --- /dev/null +++ b/tests/e2e/client_apps/codex/vision/test_bedrock_mantle.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, assert_cell, vision +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("bedrock_mantle") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.bedrock_mantle.vision.stream.codex_vision") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.BEDROCK_MANTLE,), + models=MODELS, + capabilities=(Capability.VISION,), + mode=Mode.STREAM, + ) +) +def test_vision_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, vision) diff --git a/tests/e2e/client_apps/codex/vision/test_openai.py b/tests/e2e/client_apps/codex/vision/test_openai.py new file mode 100644 index 00000000000..0775554de97 --- /dev/null +++ b/tests/e2e/client_apps/codex/vision/test_openai.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, assert_cell, vision +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("openai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.openai.vision.stream.codex_vision") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=MODELS, + capabilities=(Capability.VISION,), + mode=Mode.STREAM, + ) +) +def test_vision_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, vision) diff --git a/tests/e2e/client_apps/codex/vision/test_vertex_ai.py b/tests/e2e/client_apps/codex/vision/test_vertex_ai.py new file mode 100644 index 00000000000..91c3898d55c --- /dev/null +++ b/tests/e2e/client_apps/codex/vision/test_vertex_ai.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, assert_cell, vision +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("vertex_ai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.vertex.vision.stream.codex_vision") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.VERTEX_AI,), + models=MODELS, + capabilities=(Capability.VISION,), + mode=Mode.STREAM, + ) +) +def test_vision_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, vision) diff --git a/tests/e2e/client_apps/codex/web_search/__init__.py b/tests/e2e/client_apps/codex/web_search/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/client_apps/codex/web_search/test_azure.py b/tests/e2e/client_apps/codex/web_search/test_azure.py new file mode 100644 index 00000000000..e390b8a6bc2 --- /dev/null +++ b/tests/e2e/client_apps/codex/web_search/test_azure.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, assert_cell, web_search +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("azure") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.azure_openai.web_search.stream.codex_web_search") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.AZURE,), + models=MODELS, + capabilities=(Capability.WEB_SEARCH,), + mode=Mode.STREAM, + ) +) +def test_web_search_azure(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, web_search) diff --git a/tests/e2e/client_apps/codex/web_search/test_bedrock_mantle.py b/tests/e2e/client_apps/codex/web_search/test_bedrock_mantle.py new file mode 100644 index 00000000000..9990a9308d3 --- /dev/null +++ b/tests/e2e/client_apps/codex/web_search/test_bedrock_mantle.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, assert_cell, web_search +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("bedrock_mantle") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.bedrock_mantle.web_search.stream.codex_web_search") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.BEDROCK_MANTLE,), + models=MODELS, + capabilities=(Capability.WEB_SEARCH,), + mode=Mode.STREAM, + ) +) +def test_web_search_bedrock_mantle(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, web_search) diff --git a/tests/e2e/client_apps/codex/web_search/test_openai.py b/tests/e2e/client_apps/codex/web_search/test_openai.py new file mode 100644 index 00000000000..4374d47aee9 --- /dev/null +++ b/tests/e2e/client_apps/codex/web_search/test_openai.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, assert_cell, web_search +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("openai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.openai.web_search.stream.codex_web_search") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=MODELS, + capabilities=(Capability.WEB_SEARCH,), + mode=Mode.STREAM, + ) +) +def test_web_search_openai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, web_search) diff --git a/tests/e2e/client_apps/codex/web_search/test_vertex_ai.py b/tests/e2e/client_apps/codex/web_search/test_vertex_ai.py new file mode 100644 index 00000000000..e851debf14d --- /dev/null +++ b/tests/e2e/client_apps/codex/web_search/test_vertex_ai.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta + +from client_apps.codex.columns import column_models +from client_apps.codex.features import CodexCell, assert_cell, web_search +from client_apps.compat_collector import CompatRecorder + +MODELS: Final = column_models("vertex_ai") + + +@pytest.mark.e2e +@pytest.mark.covers("llm.responses.vertex.web_search.stream.codex_web_search") +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.VERTEX_AI,), + models=MODELS, + capabilities=(Capability.WEB_SEARCH,), + mode=Mode.STREAM, + ) +) +def test_web_search_vertex_ai(compat_result: CompatRecorder, codex_cell: CodexCell) -> None: + assert_cell(compat_result, codex_cell, MODELS, web_search) diff --git a/tests/e2e/client_apps/compat_collector.py b/tests/e2e/client_apps/compat_collector.py new file mode 100644 index 00000000000..b4af6aa67d7 --- /dev/null +++ b/tests/e2e/client_apps/compat_collector.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from functools import cache +from itertools import chain +from pathlib import Path +from typing import Final + +import pytest + +from client_apps.compat_results import ( + Failed, + NotApplicable, + Outcome, + Passed, + ResultRow, + merged_rows, + results_path, + shard_dir, + write_rows, +) +from client_apps.matrix_builder import Manifest, load_manifest + +CLIENT_APPS_DIR: Final = Path(__file__).resolve().parent +ROWS: Final = pytest.StashKey[tuple[ResultRow, ...]]() +NO_RESULT_ERROR: Final = "the test passed without reporting a result" + + +@dataclass(frozen=True, slots=True) +class CellAddress: + client: str + feature_id: str + provider: str + + +@cache +def client_manifest(client_dir: Path) -> Manifest: + return load_manifest(client_dir / "manifest.yaml") + + +def cell_address(test_file: Path, clients_dir: Path = CLIENT_APPS_DIR) -> CellAddress | None: + resolved: Final = test_file.resolve() + if not resolved.is_relative_to(clients_dir): + return None + match resolved.relative_to(clients_dir).parts: + case (client, feature_id, filename) if filename.startswith("test_") and filename.endswith(".py"): + return _manifest_cell(clients_dir / client, feature_id, filename.removeprefix("test_").removesuffix(".py")) + case _: + return None + + +def _manifest_cell(client_dir: Path, feature_id: str, provider: str) -> CellAddress | None: + if not (client_dir / "manifest.yaml").is_file(): + return None + manifest: Final = client_manifest(client_dir) + known_feature: Final = any(feature.id == feature_id for feature in manifest.features) + if not known_feature or provider not in manifest.providers: + return None + return CellAddress(client=client_dir.name, feature_id=feature_id, provider=provider) + + +def _row(address: CellAddress, model: str | None, outcome: Outcome) -> ResultRow: + return ResultRow(feature_id=address.feature_id, provider=address.provider, model=model, outcome=outcome) + + +def _record(item: pytest.Item, row: ResultRow) -> None: + item.stash[ROWS] = (*item.stash.get(ROWS, ()), row) + + +class CompatRecorder: + __slots__ = ("_item", "address") + + def __init__(self, item: pytest.Item, address: CellAddress) -> None: + self._item: Final = item + self.address: Final = address + + def passed(self, model: str) -> None: + _record(self._item, _row(self.address, model, Passed())) + + def failed(self, model: str, error: str) -> None: + _record(self._item, _row(self.address, model, Failed(error=error))) + + def not_applicable(self, reason: str) -> None: + _record(self._item, _row(self.address, None, NotApplicable(reason=reason))) + + +def _fails_cell(report: pytest.TestReport) -> bool: + return report.failed and report.when in ("setup", "call") + + +def outcome_rows( + address: CellAddress, recorded: tuple[ResultRow, ...], report: pytest.TestReport +) -> tuple[ResultRow, ...]: + if _fails_cell(report) and not any(isinstance(row.outcome, Failed) for row in recorded): + return (_row(address, None, Failed(error=_crash_message(report))),) + if report.passed and report.when == "call" and not recorded: + return (_row(address, None, Failed(error=NO_RESULT_ERROR)),) + return () + + +def _crash_message(report: pytest.TestReport) -> str: + lines: Final = tuple(line for line in report.longreprtext.splitlines() if line.strip()) + error_lines: Final = tuple(line for line in lines if line.startswith("E ")) + return (error_lines or lines or (f"{report.when} failed",))[-1].removeprefix("E ").strip() + + +def known_clients() -> tuple[str, ...]: + return tuple(sorted(manifest.parent.name for manifest in CLIENT_APPS_DIR.glob("*/manifest.yaml"))) + + +def worker_id() -> str | None: + return os.environ.get("PYTEST_XDIST_WORKER") + + +def _item_rows(item: pytest.Item, client: str) -> tuple[ResultRow, ...]: + address: Final = cell_address(item.path) + return item.stash.get(ROWS, ()) if address is not None and address.client == client else () + + +def client_rows(session: pytest.Session, client: str) -> tuple[ResultRow, ...]: + return tuple(chain.from_iterable(_item_rows(item, client) for item in session.items)) + + +def clear_shards(client: str) -> None: + shards: Final = shard_dir(results_path(client, os.environ)) + for stale in shards.glob("*.json"): + stale.unlink() + + +def publish(client: str, rows: tuple[ResultRow, ...], worker: str | None) -> None: + results: Final = results_path(client, os.environ) + if worker is not None: + if rows: + write_rows(shard_dir(results) / f"{worker}.json", rows) + return + merged: Final = merged_rows(rows, shard_dir(results)) + if merged: + write_rows(results, merged) diff --git a/tests/e2e/client_apps/compat_results.py b/tests/e2e/client_apps/compat_results.py new file mode 100644 index 00000000000..7a2fed79ef3 --- /dev/null +++ b/tests/e2e/client_apps/compat_results.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import Path +from typing import Annotated, Final, Literal + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter + + +class Passed(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + status: Literal["pass"] = "pass" + + +class Failed(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + status: Literal["fail"] = "fail" + error: str = Field(min_length=1) + + +class NotApplicable(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + status: Literal["not_applicable"] = "not_applicable" + reason: str = Field(min_length=1) + + +class NotTested(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + status: Literal["not_tested"] = "not_tested" + + +type Outcome = Annotated[Passed | Failed | NotApplicable | NotTested, Field(discriminator="status")] + + +class ResultRow(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + feature_id: str + provider: str + model: str | None + outcome: Outcome + + +RESULT_ROWS: Final = TypeAdapter(tuple[ResultRow, ...]) + + +def read_rows(path: Path) -> tuple[ResultRow, ...]: + return RESULT_ROWS.validate_json(path.read_bytes()) + + +def write_rows(path: Path, rows: tuple[ResultRow, ...]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_bytes(RESULT_ROWS.dump_json(sort_rows(rows), indent=2)) + + +def sort_rows(rows: tuple[ResultRow, ...]) -> tuple[ResultRow, ...]: + return tuple(sorted(rows, key=lambda row: (row.feature_id, row.provider, row.model or ""))) + + +def results_path(client: str, environ: Mapping[str, str]) -> Path: + return Path(environ.get(f"{client.upper()}_COMPAT_RESULTS_PATH", f"{client}-compat-results.json")) + + +def shard_dir(results: Path) -> Path: + return results.with_name(f"{results.name}.shards") + + +def merged_rows(own: tuple[ResultRow, ...], shards: Path) -> tuple[ResultRow, ...]: + shard_files: Final = tuple(sorted(shards.glob("*.json"))) if shards.is_dir() else () + return sort_rows((*own, *(row for shard in shard_files for row in read_rows(shard)))) diff --git a/tests/e2e/client_apps/conftest.py b/tests/e2e/client_apps/conftest.py new file mode 100644 index 00000000000..1c8d6233d05 --- /dev/null +++ b/tests/e2e/client_apps/conftest.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from collections.abc import Generator +from typing import Final + +import pytest + +from client_apps.compat_collector import ( + ROWS, + CompatRecorder, + cell_address, + clear_shards, + client_rows, + known_clients, + outcome_rows, + publish, + worker_id, +) + + +@pytest.fixture +def compat_result(request: pytest.FixtureRequest) -> CompatRecorder: + item: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untyped + assert isinstance(item, pytest.Item) + address: Final = cell_address(item.path) + if address is None: + pytest.fail(f"{item.path} is not a //test_.py cell of a client manifest") + return CompatRecorder(item, address) + + +@pytest.hookimpl(tryfirst=True) +def pytest_runtest_setup(item: pytest.Item) -> None: + item.stash[ROWS] = () + + +@pytest.hookimpl(wrapper=True) +def pytest_runtest_makereport( + item: pytest.Item, call: pytest.CallInfo[None] +) -> Generator[None, pytest.TestReport, pytest.TestReport]: + report: Final = yield + address: Final = cell_address(item.path) + if address is None: + return report + extra: Final = outcome_rows(address, item.stash.get(ROWS, ()), report) + if extra: + item.stash[ROWS] = (*item.stash.get(ROWS, ()), *extra) + return report + + +def pytest_sessionstart(session: pytest.Session) -> None: + if worker_id() is not None: + return + for client in known_clients(): + clear_shards(client) + + +def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: + for client in known_clients(): + publish(client, client_rows(session, client), worker_id()) diff --git a/tests/e2e/client_apps/matrix_builder.py b/tests/e2e/client_apps/matrix_builder.py new file mode 100644 index 00000000000..f50a404f351 --- /dev/null +++ b/tests/e2e/client_apps/matrix_builder.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +import sys +from argparse import ArgumentParser +from collections.abc import Iterator, Mapping +from datetime import datetime, timezone +from importlib.metadata import version +from pathlib import Path +from types import MappingProxyType +from typing import Annotated, Final, Literal + +import yaml +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter + +from client_apps.compat_results import ( + Failed, + NotApplicable, + NotTested, + Outcome, + Passed, + ResultRow, + read_rows, +) + +CLIENT_APPS_DIR: Final = Path(__file__).resolve().parent +REGRESSION_EXIT_CODE: Final = 3 + + +class _Frozen(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + +class ManifestFeature(_Frozen): + id: str + name: str + + +class Manifest(_Frozen): + schema_version: str + client: str + package: str + providers: tuple[str, ...] + features: tuple[ManifestFeature, ...] + + +class MatrixFeature(_Frozen): + id: str + name: str + providers: Mapping[str, Outcome] + + +class Matrix(_Frozen): + schema_version: str + generated_at: str + litellm_version: str + client: str + client_version: str + providers: tuple[str, ...] + features: tuple[MatrixFeature, ...] + + +class Regression(_Frozen): + feature_id: str + feature_name: str + provider: str + error: str + + +class _PackageJson(BaseModel): + model_config = ConfigDict(frozen=True) + + dependencies: Mapping[str, str] + + +class BuildArgs(_Frozen): + command: Literal["build"] + manifest: Path + results: Path + out: Path + litellm_version: str | None + + +class RegressionArgs(_Frozen): + command: Literal["regressions"] + old: Path + new: Path + + +CLI_ARGS: Final = TypeAdapter[BuildArgs | RegressionArgs]( + Annotated[BuildArgs | RegressionArgs, Field(discriminator="command")] +) +REGRESSIONS: Final = TypeAdapter(tuple[Regression, ...]) + + +def load_manifest(path: Path) -> Manifest: + return Manifest.model_validate(yaml.safe_load(path.read_text())) + + +def pinned_client_version(package: str, package_json: Path = CLIENT_APPS_DIR / "package.json") -> str: + return _PackageJson.model_validate_json(package_json.read_bytes()).dependencies[package] + + +def aggregate(rows: tuple[ResultRow, ...]) -> Outcome: + tested: Final = tuple(row for row in rows if not isinstance(row.outcome, NotTested)) + failures: Final = tuple(_failure(row.model, row.outcome) for row in tested if isinstance(row.outcome, Failed)) + if failures: + return Failed(error="; ".join(failures)) + if any(isinstance(row.outcome, Passed) for row in tested): + return Passed() + reasons: Final = tuple(row.outcome.reason for row in tested if isinstance(row.outcome, NotApplicable)) + if reasons: + return NotApplicable(reason=reasons[0]) + return NotTested() + + +def _failure(model: str | None, outcome: Failed) -> str: + return outcome.error if model is None else f"{model}: {outcome.error}" + + +def build_matrix( + manifest: Manifest, + rows: tuple[ResultRow, ...], + *, + litellm_version: str, + client_version: str, + generated_at: datetime, +) -> Matrix: + return Matrix( + schema_version=manifest.schema_version, + generated_at=generated_at.isoformat(), + litellm_version=litellm_version, + client=manifest.client, + client_version=client_version, + providers=manifest.providers, + features=tuple( + MatrixFeature( + id=feature.id, + name=feature.name, + providers=MappingProxyType( + {provider: aggregate(_cell_rows(rows, feature.id, provider)) for provider in manifest.providers} + ), + ) + for feature in manifest.features + ), + ) + + +def _cell_rows(rows: tuple[ResultRow, ...], feature_id: str, provider: str) -> tuple[ResultRow, ...]: + return tuple(row for row in rows if row.feature_id == feature_id and row.provider == provider) + + +def _cells(matrix: Matrix) -> Iterator[tuple[MatrixFeature, str, Outcome]]: + for feature in matrix.features: + for provider, outcome in feature.providers.items(): + yield feature, provider, outcome + + +def find_regressions(old: Matrix, new: Matrix) -> tuple[Regression, ...]: + old_cells: Final = MappingProxyType({(feature.id, provider): outcome for feature, provider, outcome in _cells(old)}) + return tuple( + sorted( + ( + Regression(feature_id=feature.id, feature_name=feature.name, provider=provider, error=outcome.error) + for feature, provider, outcome in _cells(new) + if isinstance(outcome, Failed) and isinstance(old_cells.get((feature.id, provider)), Passed) + ), + key=lambda regression: (regression.feature_id, regression.provider), + ) + ) + + +def _build(args: BuildArgs) -> int: + manifest: Final = load_manifest(args.manifest) + matrix: Final = build_matrix( + manifest, + read_rows(args.results), + litellm_version=args.litellm_version or version("litellm"), + client_version=pinned_client_version(manifest.package), + generated_at=datetime.now(timezone.utc), + ) + args.out.parent.mkdir(parents=True, exist_ok=True) + _ = args.out.write_text(matrix.model_dump_json(indent=2)) + return 0 + + +def _regressions(args: RegressionArgs) -> int: + if not args.old.exists(): + return 0 + regressions: Final = find_regressions( + Matrix.model_validate_json(args.old.read_bytes()), Matrix.model_validate_json(args.new.read_bytes()) + ) + _ = sys.stdout.write(REGRESSIONS.dump_json(regressions, indent=2).decode() + "\n") + return REGRESSION_EXIT_CODE if regressions else 0 + + +def main(argv: tuple[str, ...]) -> int: + parser: Final = ArgumentParser(prog="python -m client_apps.matrix_builder") + commands: Final = parser.add_subparsers(dest="command", required=True) + build: Final = commands.add_parser("build") + _ = build.add_argument("--manifest", required=True) + _ = build.add_argument("--results", required=True) + _ = build.add_argument("--out", required=True) + _ = build.add_argument("--litellm-version", dest="litellm_version", default=None) + regressions: Final = commands.add_parser("regressions") + _ = regressions.add_argument("--old", required=True) + _ = regressions.add_argument("--new", required=True) + match CLI_ARGS.validate_python(vars(parser.parse_args(argv))): + case BuildArgs() as build_args: + return _build(build_args) + case RegressionArgs() as regression_args: + return _regressions(regression_args) + + +if __name__ == "__main__": + raise SystemExit(main(tuple(sys.argv[1:]))) diff --git a/tests/e2e/client_apps/package-lock.json b/tests/e2e/client_apps/package-lock.json new file mode 100644 index 00000000000..6e67c306b7d --- /dev/null +++ b/tests/e2e/client_apps/package-lock.json @@ -0,0 +1,137 @@ +{ + "name": "litellm-client-apps-e2e", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "litellm-client-apps-e2e", + "version": "0.0.0", + "dependencies": { + "@openai/codex": "0.164.0" + } + }, + "node_modules/@openai/codex": { + "version": "0.164.0", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0.tgz", + "integrity": "sha512-TWj7UGyPRe3Pj/A3ddNAxs1te3a/W/xAd+J0LHrrFTmsSfahGMOLF/vn88D3QCIgAT3B1rm1y/Z3NYE0rx6cZQ==", + "license": "Apache-2.0", + "bin": { + "codex": "bin/codex.js" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@openai/codex-darwin-arm64": "npm:@openai/codex@0.164.0-darwin-arm64", + "@openai/codex-darwin-x64": "npm:@openai/codex@0.164.0-darwin-x64", + "@openai/codex-linux-arm64": "npm:@openai/codex@0.164.0-linux-arm64", + "@openai/codex-linux-x64": "npm:@openai/codex@0.164.0-linux-x64", + "@openai/codex-win32-arm64": "npm:@openai/codex@0.164.0-win32-arm64", + "@openai/codex-win32-x64": "npm:@openai/codex@0.164.0-win32-x64" + } + }, + "node_modules/@openai/codex-darwin-arm64": { + "name": "@openai/codex", + "version": "0.164.0-darwin-arm64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-darwin-arm64.tgz", + "integrity": "sha512-pDizgOUAtq3/k3AbUFHbaU19WpYKCja+oFpNSTLKrDQj4lTUNSDuTPJip4uQVn2/1EipQdpQrGMOQBVuI1znAA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@openai/codex-darwin-x64": { + "name": "@openai/codex", + "version": "0.164.0-darwin-x64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-darwin-x64.tgz", + "integrity": "sha512-lIe+udnguPGzBezkgbkj2Eaz0YqOhNXT7jja136Fj0sXfI+UKNS2icmaK14GcoRAgeVEpDTtC1ftamWBPmD1zg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@openai/codex-linux-arm64": { + "name": "@openai/codex", + "version": "0.164.0-linux-arm64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-linux-arm64.tgz", + "integrity": "sha512-LL5iTQjyDL4YRX44EWqhIfEQ5uGnmgxkQBJSxjZSeJ8L1Og83pdlzUJW/PRJ76oaVwOVgh6HXwuPEiBNEHU4lg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@openai/codex-linux-x64": { + "name": "@openai/codex", + "version": "0.164.0-linux-x64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-linux-x64.tgz", + "integrity": "sha512-XLAoaWsGOT2khzl+P4FYDPTkzAr9W0VFXFaVkqlFdkM0soa1gH/4S0wSD57Ek3HHwkirAEGz2/Mwehseh+l5Mg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@openai/codex-win32-arm64": { + "name": "@openai/codex", + "version": "0.164.0-win32-arm64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-win32-arm64.tgz", + "integrity": "sha512-Zkzt8HNmB/vOnlDo5D7IOlrt5rHaxYQB7iGbNP7bHcXZtrk6gaD7UriwaieGYJpcf8DqAT/0tqzY4xbs8ZwbmQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@openai/codex-win32-x64": { + "name": "@openai/codex", + "version": "0.164.0-win32-x64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.164.0-win32-x64.tgz", + "integrity": "sha512-urWPhwh9YavuoAPIkmVYKBTtd5OtGVL4Ki0Fb28dhK/Mz6Pt4LSy+gbRrZFz5+RXrw9ubvXuXDCcd8beYcWhDg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + } + } +} diff --git a/tests/e2e/client_apps/package.json b/tests/e2e/client_apps/package.json new file mode 100644 index 00000000000..8c4ea263877 --- /dev/null +++ b/tests/e2e/client_apps/package.json @@ -0,0 +1,8 @@ +{ + "name": "litellm-client-apps-e2e", + "version": "0.0.0", + "private": true, + "dependencies": { + "@openai/codex": "0.164.0" + } +} diff --git a/tests/e2e/coverage_registry/llm_codex_compat.yaml b/tests/e2e/coverage_registry/llm_codex_compat.yaml new file mode 100644 index 00000000000..18d1dca5e70 --- /dev/null +++ b/tests/e2e/coverage_registry/llm_codex_compat.yaml @@ -0,0 +1,40 @@ +# Codex compatibility matrix: /v1/responses coverage for the Codex CLI (tests/e2e/client_apps/codex) across +# the four provider columns it drives. Each row is one (feature x provider) cell. +# +# Grammar: llm.responses...stream.codex_ +- {id: llm.responses.openai.basic.stream.codex_basic_messaging, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [codex_basic_messaging], source: "codex compat matrix", rationale: "Codex replies through the proxy"} +- {id: llm.responses.azure_openai.basic.stream.codex_basic_messaging, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: basic, streaming: stream, assertions: [codex_basic_messaging], source: "codex compat matrix", rationale: "Codex replies through the proxy"} +- {id: llm.responses.bedrock_mantle.basic.stream.codex_basic_messaging, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: basic, streaming: stream, assertions: [codex_basic_messaging], source: "codex compat matrix", rationale: "Codex replies through the proxy"} +- {id: llm.responses.vertex.basic.stream.codex_basic_messaging, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: basic, streaming: stream, assertions: [codex_basic_messaging], source: "codex compat matrix", rationale: "Codex replies through the proxy"} +- {id: llm.responses.openai.tool_use.stream.codex_shell_command, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: tool_use, streaming: stream, assertions: [codex_shell_command], source: "codex compat matrix", rationale: "Codex runs a shell command with its exec tool through the proxy"} +- {id: llm.responses.azure_openai.tool_use.stream.codex_shell_command, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: tool_use, streaming: stream, assertions: [codex_shell_command], source: "codex compat matrix", rationale: "Codex runs a shell command with its exec tool through the proxy"} +- {id: llm.responses.bedrock_mantle.tool_use.stream.codex_shell_command, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: tool_use, streaming: stream, assertions: [codex_shell_command], source: "codex compat matrix", rationale: "Codex runs a shell command with its exec tool through the proxy"} +- {id: llm.responses.vertex.tool_use.stream.codex_shell_command, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: tool_use, streaming: stream, assertions: [codex_shell_command], source: "codex compat matrix", rationale: "Codex runs a shell command with its exec tool through the proxy"} +- {id: llm.responses.openai.tool_use.stream.codex_file_edit, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: tool_use, streaming: stream, assertions: [codex_file_edit], source: "codex compat matrix", rationale: "Codex edits a file with its apply_patch tool through the proxy"} +- {id: llm.responses.azure_openai.tool_use.stream.codex_file_edit, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: tool_use, streaming: stream, assertions: [codex_file_edit], source: "codex compat matrix", rationale: "Codex edits a file with its apply_patch tool through the proxy"} +- {id: llm.responses.bedrock_mantle.tool_use.stream.codex_file_edit, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: tool_use, streaming: stream, assertions: [codex_file_edit], source: "codex compat matrix", rationale: "Codex edits a file with its apply_patch tool through the proxy"} +- {id: llm.responses.vertex.tool_use.stream.codex_file_edit, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: tool_use, streaming: stream, assertions: [codex_file_edit], source: "codex compat matrix", rationale: "Codex edits a file with apply_patch through the proxy, which Gemini calls through its exec tool"} +- {id: llm.responses.openai.multi_turn.stream.codex_multi_turn, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: multi_turn, streaming: stream, assertions: [codex_multi_turn], source: "codex compat matrix", rationale: "A resumed Codex thread recalls an earlier turn through the proxy"} +- {id: llm.responses.azure_openai.multi_turn.stream.codex_multi_turn, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: multi_turn, streaming: stream, assertions: [codex_multi_turn], source: "codex compat matrix", rationale: "A resumed Codex thread recalls an earlier turn through the proxy"} +- {id: llm.responses.bedrock_mantle.multi_turn.stream.codex_multi_turn, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: multi_turn, streaming: stream, assertions: [codex_multi_turn], source: "codex compat matrix", rationale: "A resumed Codex thread recalls an earlier turn through the proxy"} +- {id: llm.responses.vertex.multi_turn.stream.codex_multi_turn, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: multi_turn, streaming: stream, assertions: [codex_multi_turn], source: "codex compat matrix", rationale: "A resumed Codex thread recalls an earlier turn through the proxy"} +- {id: llm.responses.openai.prompt_cache_5m.stream.codex_prompt_caching, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: prompt_cache_5m, streaming: stream, assertions: [codex_prompt_caching], source: "codex compat matrix", rationale: "The second turn of a Codex thread reports cached input tokens"} +- {id: llm.responses.azure_openai.prompt_cache_5m.stream.codex_prompt_caching, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: prompt_cache_5m, streaming: stream, assertions: [codex_prompt_caching], source: "codex compat matrix", rationale: "The second turn of a Codex thread reports cached input tokens"} +- {id: llm.responses.bedrock_mantle.prompt_cache_5m.stream.codex_prompt_caching, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: prompt_cache_5m, streaming: stream, assertions: [codex_prompt_caching], source: "codex compat matrix", rationale: "The second turn of a Codex thread reports cached input tokens"} +- {id: llm.responses.vertex.prompt_cache_5m.stream.codex_prompt_caching, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: prompt_cache_5m, streaming: stream, assertions: [codex_prompt_caching], source: "codex compat matrix", rationale: "The second turn of a Codex thread reports cached input tokens"} +- {id: llm.responses.openai.thinking.stream.codex_reasoning_effort, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: thinking, streaming: stream, assertions: [codex_reasoning_effort], source: "codex compat matrix", rationale: "Codex model_reasoning_effort=high reaches the model and reasoning tokens come back"} +- {id: llm.responses.azure_openai.thinking.stream.codex_reasoning_effort, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: thinking, streaming: stream, assertions: [codex_reasoning_effort], source: "codex compat matrix", rationale: "Codex model_reasoning_effort=high reaches the model and reasoning tokens come back"} +- {id: llm.responses.bedrock_mantle.thinking.stream.codex_reasoning_effort, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: thinking, streaming: stream, assertions: [codex_reasoning_effort], source: "codex compat matrix", rationale: "Codex model_reasoning_effort=high reaches the model and reasoning tokens come back"} +- {id: llm.responses.vertex.thinking.stream.codex_reasoning_effort, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: thinking, streaming: stream, assertions: [codex_reasoning_effort], source: "codex compat matrix", rationale: "Codex model_reasoning_effort=high reaches the model and reasoning tokens come back"} +- {id: llm.responses.openai.vision.stream.codex_vision, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: vision, streaming: stream, assertions: [codex_vision], source: "codex compat matrix", rationale: "Codex image input reaches the model through the proxy"} +- {id: llm.responses.azure_openai.vision.stream.codex_vision, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: vision, streaming: stream, assertions: [codex_vision], source: "codex compat matrix", rationale: "Codex image input reaches the model through the proxy"} +- {id: llm.responses.bedrock_mantle.vision.stream.codex_vision, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: vision, streaming: stream, assertions: [codex_vision], source: "codex compat matrix", rationale: "Codex image input reaches the model through the proxy"} +- {id: llm.responses.vertex.vision.stream.codex_vision, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: vision, streaming: stream, assertions: [codex_vision], source: "codex compat matrix", rationale: "Codex image input reaches the model through the proxy"} +- {id: llm.responses.openai.structured_output.stream.codex_structured_output, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: structured_output, streaming: stream, assertions: [codex_structured_output], source: "codex compat matrix", rationale: "Codex --output-schema yields JSON matching the schema"} +- {id: llm.responses.azure_openai.structured_output.stream.codex_structured_output, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: structured_output, streaming: stream, assertions: [codex_structured_output], source: "codex compat matrix", rationale: "Codex --output-schema yields JSON matching the schema"} +- {id: llm.responses.bedrock_mantle.structured_output.stream.codex_structured_output, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: structured_output, streaming: stream, assertions: [codex_structured_output], source: "codex compat matrix", rationale: "Codex --output-schema yields JSON matching the schema"} +- {id: llm.responses.vertex.structured_output.stream.codex_structured_output, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: structured_output, streaming: stream, assertions: [codex_structured_output], source: "codex compat matrix", rationale: "Codex --output-schema yields JSON matching the schema"} +- {id: llm.responses.openai.web_search.stream.codex_web_search, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: web_search, streaming: stream, assertions: [codex_web_search], source: "codex compat matrix", rationale: "Codex web search runs through the proxy"} +- {id: llm.responses.azure_openai.web_search.stream.codex_web_search, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: web_search, streaming: stream, assertions: [codex_web_search], source: "codex compat matrix", rationale: "Codex web search runs through the proxy"} +- {id: llm.responses.bedrock_mantle.web_search.stream.codex_web_search, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_mantle, capability: web_search, streaming: stream, assertions: [codex_web_search], source: "codex compat matrix", rationale: "Codex web search runs through the proxy"} +- {id: llm.responses.vertex.web_search.stream.codex_web_search, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: web_search, streaming: stream, assertions: [codex_web_search], source: "codex compat matrix", rationale: "Codex web search runs through the proxy"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index d2ba2c325df..7fd44ff502c 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -52,6 +52,7 @@ LlmRoute = Literal[ "azure_openai", "bedrock_converse", "bedrock_invoke", + "bedrock_mantle", "cohere", "gemini", "hosted_vllm", diff --git a/tests/e2e_harness/AGENTS.md b/tests/e2e_harness/AGENTS.md index 65a3b32df96..9bfa2898095 100644 --- a/tests/e2e_harness/AGENTS.md +++ b/tests/e2e_harness/AGENTS.md @@ -1,8 +1,8 @@ # e2e harness tests -Tests of the harness under `tests/e2e/` (the transport, the clients, the fixture bundle and replay edge, the stack lock, the IdP launcher, the coverage collector, the JUnit properties and the Claude Code driver), not of the product. They live outside `tests/e2e/` because the Buildkite e2e run copies that folder into the runner image and runs every test in it, so a harness test in there counts as a product test in the nightly numbers. Nothing here needs a proxy, provider keys or the network +Tests of the harness under `tests/e2e/` (the transport, the clients, the fixture bundle and replay edge, the stack lock, the IdP launcher, the coverage collector, the JUnit properties, the Claude Code driver and the client compatibility matrices), not of the product. They live outside `tests/e2e/` because the Buildkite e2e run copies that folder into the runner image and runs every test in it, so a harness test in there counts as a product test in the nightly numbers. Nothing here needs a proxy, provider keys or the network -The layout mirrors `tests/e2e/`: `test_e2e_http.py` covers `tests/e2e/e2e_http.py`, `logging/test_datadog_reader.py` covers `tests/e2e/logging/datadog_reader.py`, and `claude_code/` covers the driver, builder, probe and version resolver. Put a new harness test under the folder that mirrors the suite folder whose module it covers +The layout mirrors `tests/e2e/`: `test_e2e_http.py` covers `tests/e2e/e2e_http.py`, `logging/test_datadog_reader.py` covers `tests/e2e/logging/datadog_reader.py`, `claude_code/` covers the driver, builder, probe and version resolver, and `client_apps/` covers the client compatibility matrices' collector and builder, with `client_apps/codex/` covering the Codex event parser (against `codex exec --json` transcripts recorded from live runs), the CLI driver and the column lock. Put a new harness test under the folder that mirrors the suite folder whose module it covers Run them from the repo root. `e2e_config` reads `LITELLM_MASTER_KEY` at import and any value will do, the CI lane sets a dummy: diff --git a/tests/e2e_harness/client_apps/codex/test_codex_cli_driver.py b/tests/e2e_harness/client_apps/codex/test_codex_cli_driver.py new file mode 100644 index 00000000000..aa7e4b9a859 --- /dev/null +++ b/tests/e2e_harness/client_apps/codex/test_codex_cli_driver.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Final + +import pytest + +from client_apps.codex.cli_driver import ( + LITELLM_PROVIDER_KEY_ENV, + OPENAI_PROVIDER_KEY_ENV, + TIMEOUT_ENV, + CodexTurn, + Finished, + Gateway, + TimedOut, + Wiring, + command, + environment, + new_session, + resolve_binary, + run_codex, + stderr_tail, + timeout_seconds, +) + +GATEWAY: Final = Gateway(base_url="http://127.0.0.1:24001", api_key="sk-scoped") +EMITTING_STUB: Final = """#!/bin/sh +printf '%s\\n' "$@" >&2 +echo '{"type":"thread.started","thread_id":"t-1"}' +echo '{"type":"item.completed","item":{"id":"i","type":"agent_message","text":"hi"}}' +echo '{"type":"turn.completed","usage":{"input_tokens":3,"output_tokens":1}}' +exit 4 +""" +HANGING_STUB: Final = """#!/bin/sh +echo '{"type":"thread.started","thread_id":"t-2"}' +sleep 30 +""" + + +def _stub(directory: Path, script: str) -> Path: + binary: Final = directory / "codex" + _ = binary.write_text(script) + binary.chmod(0o755) + return binary + + +def _option(argv: tuple[str, ...], name: str) -> tuple[str, ...]: + return tuple(argv[index + 1] for index, value in enumerate(argv[:-1]) if value == name) + + +def test_litellm_provider_command_points_codex_at_the_proxy_v1_responses(tmp_path: Path) -> None: + session: Final = new_session(tmp_path) + argv: Final = command(Path("/bin/codex"), GATEWAY, session, CodexTurn("gpt-6.1-sol", "-- say hi")) + + assert argv[:2] == ("/bin/codex", "exec") + assert set(_option(argv, "-c")) >= { + 'model_provider="litellm"', + 'model_providers.litellm.base_url="http://127.0.0.1:24001/v1"', + f'model_providers.litellm.env_key="{LITELLM_PROVIDER_KEY_ENV}"', + 'model_providers.litellm.wire_api="responses"', + 'approval_policy="never"', + } + assert _option(argv, "-m") == ("gpt-6.1-sol",) + assert _option(argv, "-C") == (str(session.workdir),) + assert argv[-2:] == ("--", "-- say hi") + + +def test_resume_comes_after_every_exec_option_and_before_the_prompt(tmp_path: Path) -> None: + turn: Final = CodexTurn("gpt-6.1-sol", "again", options=("-i", "img.png"), resume_thread="t-9") + argv: Final = command(Path("codex"), GATEWAY, new_session(tmp_path), turn) + resume_at: Final = argv.index("resume") + + assert argv[resume_at : resume_at + 4] == ("resume", "t-9", "--", "again") + assert argv.index("-i") < resume_at + assert max(index for index, value in enumerate(argv) if value == "-c") < resume_at + + +def test_openai_base_url_wiring_keeps_the_builtin_provider(tmp_path: Path) -> None: + turn: Final = CodexTurn("gpt-6.1-sol", "p", wiring=Wiring.OPENAI_BASE_URL) + overrides: Final = _option(command(Path("codex"), GATEWAY, new_session(tmp_path), turn), "-c") + + assert 'openai_base_url="http://127.0.0.1:24001/v1"' in overrides + assert not any(override.startswith("model_provider") for override in overrides) + + +def test_environment_carries_only_the_session_dirs_and_the_wired_key( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-leak") + session: Final = new_session(tmp_path) + litellm_env: Final = environment(GATEWAY, session, Wiring.LITELLM_PROVIDER) + openai_env: Final = environment(GATEWAY, session, Wiring.OPENAI_BASE_URL) + + assert set(litellm_env) == {"PATH", "HOME", "CODEX_HOME", LITELLM_PROVIDER_KEY_ENV} + assert set(openai_env) == {"PATH", "HOME", "CODEX_HOME", OPENAI_PROVIDER_KEY_ENV} + assert litellm_env[LITELLM_PROVIDER_KEY_ENV] == openai_env[OPENAI_PROVIDER_KEY_ENV] == "sk-scoped" + assert litellm_env["CODEX_HOME"] == str(session.codex_home) + assert litellm_env["HOME"] == str(session.home) + + +def test_resolve_binary_prefers_the_pinned_local_install(tmp_path: Path) -> None: + local: Final = _stub(tmp_path, "#!/bin/sh\n") + + assert resolve_binary(local, which=lambda _: "/usr/bin/codex") == local + assert resolve_binary(tmp_path / "missing", which=lambda _: "/usr/bin/codex") == Path("/usr/bin/codex") + assert resolve_binary(tmp_path / "missing", which=lambda _: None) is None + + +def test_timeout_seconds_reads_the_env_override() -> None: + assert timeout_seconds({TIMEOUT_ENV: "12.5"}) == 12.5 + assert timeout_seconds({}) > 0 + + +def test_run_codex_parses_stdout_and_keeps_each_turn_log(tmp_path: Path) -> None: + binary: Final = _stub(tmp_path, EMITTING_STUB) + session: Final = new_session(tmp_path / "session") + first: Final = run_codex(binary, GATEWAY, session, CodexTurn("m", "p"), 30) + second: Final = run_codex(binary, GATEWAY, session, CodexTurn("m", "q", resume_thread="t-1"), 30) + + assert isinstance(first, Finished) + assert first.exit_code == 4 + assert first.transcript.thread_id == "t-1" + assert first.transcript.final_message == "hi" + assert first.transcript.usage is not None + assert first.transcript.usage.input_tokens == 3 + assert isinstance(second, Finished) + assert "resume\nt-1\n--\nq" in second.stderr + assert sorted(path.name for path in session.root.glob("turn-*")) == [ + "turn-1.jsonl", + "turn-1.stderr.txt", + "turn-2.jsonl", + "turn-2.stderr.txt", + ] + + +def test_run_codex_kills_a_hung_turn_and_keeps_what_it_printed(tmp_path: Path) -> None: + run: Final = run_codex(_stub(tmp_path, HANGING_STUB), GATEWAY, new_session(tmp_path / "s"), CodexTurn("m", "p"), 1) + + assert isinstance(run, TimedOut) + assert run.transcript.thread_id == "t-2" + + +def test_stderr_tail_drops_the_stdin_notice_and_keeps_the_end() -> None: + tail: Final = stderr_tail("Reading additional input from stdin...\n\n" + "x" * 2000 + "\nboom") + + assert "Reading additional input" not in tail + assert tail.endswith("boom") + assert len(tail) <= 600 diff --git a/tests/e2e_harness/client_apps/codex/test_codex_columns.py b/tests/e2e_harness/client_apps/codex/test_codex_columns.py new file mode 100644 index 00000000000..0e6ee7eed19 --- /dev/null +++ b/tests/e2e_harness/client_apps/codex/test_codex_columns.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import threading +from itertools import chain +from pathlib import Path +from typing import Final + +from client_apps.codex.columns import column_lock, column_models, load_columns +from client_apps.compat_collector import CLIENT_APPS_DIR, client_manifest + + +def _acquire_in_thread(provider: str, lock_dir: Path) -> threading.Event: + acquired: Final = threading.Event() + + def hold() -> None: + with column_lock(provider, lock_dir): + acquired.set() + + threading.Thread(target=hold, daemon=True).start() + return acquired + + +def test_column_lock_lets_one_cell_per_column_run_at_a_time(tmp_path: Path) -> None: + with column_lock("azure", tmp_path): + same_column: Final = _acquire_in_thread("azure", tmp_path) + other_column: Final = _acquire_in_thread("openai", tmp_path) + + assert other_column.wait(5) + assert not same_column.wait(0.3) + + assert same_column.wait(5) + + +def test_every_manifest_column_has_deployments_with_unique_names() -> None: + manifest: Final = client_manifest(CLIENT_APPS_DIR / "codex") + names: Final = tuple(chain.from_iterable(column_models(provider) for provider in manifest.providers)) + + assert set(load_columns()) == set(manifest.providers) + assert all(column_models(provider) for provider in manifest.providers) + assert len(set(names)) == len(names) diff --git a/tests/e2e_harness/client_apps/codex/test_codex_events.py b/tests/e2e_harness/client_apps/codex/test_codex_events.py new file mode 100644 index 00000000000..46c3867c1d5 --- /dev/null +++ b/tests/e2e_harness/client_apps/codex/test_codex_events.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Final + +from client_apps.codex.events import ( + CommandExecution, + FileChange, + ItemError, + Reasoning, + Transcript, + UnknownEvent, + UnknownItem, + WebSearch, + parse_transcript, +) + +TRANSCRIPTS: Final = Path(__file__).resolve().parent / "transcripts" + + +def _transcript(name: str) -> Transcript: + return parse_transcript((TRANSCRIPTS / name).read_text()) + + +def test_every_recorded_line_parses() -> None: + unparsed: Final = {path.name: _transcript(path.name).unparsed for path in sorted(TRANSCRIPTS.glob("*.jsonl"))} + + assert len(unparsed) == 9 + assert all(lines == () for lines in unparsed.values()), unparsed + + +def test_file_change_lists_the_touched_path_once_completed() -> None: + transcript: Final = _transcript("file_change.jsonl") + changes: Final = tuple(item for item in transcript.completed_items if isinstance(item, FileChange)) + + assert len(changes) == 1 + assert changes[0].status == "completed" + assert [(update.path, update.kind) for update in changes[0].changes] == [ + ("/tmp/codex-session/workdir/notes.txt", "add") + ] + assert transcript.final_message == "Created `notes.txt` containing only `q7w8e9`." + assert transcript.failure is None + + +def test_shell_command_keeps_exit_code_and_output_of_the_completed_item() -> None: + transcript: Final = _transcript("shell_command.jsonl") + commands: Final = tuple(item for item in transcript.completed_items if isinstance(item, CommandExecution)) + + assert [(command.exit_code, command.aggregated_output) for command in commands] == [(0, "c887862269b3\n")] + assert transcript.final_message == "c887862269b3" + + +def test_completed_web_search_carries_its_query() -> None: + searches: Final = tuple( + item for item in _transcript("web_search.jsonl").completed_items if isinstance(item, WebSearch) + ) + + assert [search.query for search in searches] == ["LiteLLM documentation official"] + + +def test_web_search_that_only_started_is_not_a_completed_item() -> None: + transcript: Final = _transcript("web_search_never_completed.jsonl") + + assert not any(isinstance(item, WebSearch) for item in transcript.completed_items) + assert transcript.final_message == "docs.litellm.ai" + assert transcript.usage is not None + + +def test_fallback_metadata_warning_is_an_item_not_a_turn_failure() -> None: + transcript: Final = _transcript("fallback_metadata.jsonl") + errors: Final = tuple(item for item in transcript.completed_items if isinstance(item, ItemError)) + + assert len(errors) == 1 + assert "fallback metadata" in errors[0].message + assert transcript.failure is None + assert any(isinstance(item, Reasoning) and item.text for item in transcript.completed_items) + assert transcript.usage is not None + assert transcript.usage.reasoning_output_tokens == 205 + assert transcript.final_message == "15" + + +def test_turn_failed_reports_the_proxy_error_and_no_usage() -> None: + transcript: Final = _transcript("turn_failed.jsonl") + + assert transcript.failure is not None + assert "Invalid model name passed in model=gpt-6.1-sol-does-not-exist" in transcript.failure + assert transcript.usage is None + assert transcript.final_message is None + + +def test_rate_limited_turn_reports_the_final_error_not_a_retry_notice() -> None: + transcript: Final = _transcript("rate_limited.jsonl") + + assert transcript.failure is not None + assert transcript.failure.startswith("rate limit exceeded") + assert "Reconnecting" not in transcript.failure + + +def test_a_turn_that_recovers_after_a_retry_notice_has_no_failure() -> None: + transcript: Final = parse_transcript( + "\n".join( + ( + '{"type":"thread.started","thread_id":"t-1"}', + '{"type":"turn.started"}', + '{"type":"error","message":"Reconnecting... 1/5 (rate limit exceeded: 429)"}', + '{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"done"}}', + '{"type":"turn.completed","usage":{"input_tokens":3,"output_tokens":1}}', + ) + ) + ) + + assert transcript.failure is None + assert transcript.final_message == "done" + + +def test_a_stream_that_ends_on_an_error_without_turn_events_is_a_failure() -> None: + transcript: Final = parse_transcript( + '{"type":"thread.started","thread_id":"t-1"}\n{"type":"error","message":"stream disconnected"}\n' + ) + + assert transcript.failure == "stream disconnected" + + +def test_resumed_turn_keeps_the_thread_and_reports_cached_input() -> None: + first: Final = _transcript("first_turn.jsonl") + resumed: Final = _transcript("resumed_turn.jsonl") + + assert first.thread_id is not None + assert resumed.thread_id == first.thread_id + assert first.usage is not None + assert resumed.usage is not None + assert first.usage.cached_input_tokens == 0 + assert resumed.usage.cached_input_tokens == 10870 + + +def test_unknown_shapes_parse_as_unknown_and_garbage_lines_are_kept_aside() -> None: + transcript: Final = parse_transcript( + "\n".join( + ( + '{"type":"thread.started","thread_id":"t-1"}', + '{"type":"turn.compacted"}', + '{"type":"item.completed","item":{"id":"item_0","type":"collab_tool_call"}}', + '{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"done"}}', + "not json", + "", + ) + ) + ) + + assert transcript.thread_id == "t-1" + assert any(isinstance(event, UnknownEvent) and event.type == "turn.compacted" for event in transcript.events) + assert [item.type for item in transcript.completed_items if isinstance(item, UnknownItem)] == ["collab_tool_call"] + assert transcript.final_message == "done" + assert [line.text for line in transcript.unparsed] == ["not json"] diff --git a/tests/e2e_harness/client_apps/codex/transcripts/fallback_metadata.jsonl b/tests/e2e_harness/client_apps/codex/transcripts/fallback_metadata.jsonl new file mode 100644 index 00000000000..cc5eb3d2bb7 --- /dev/null +++ b/tests/e2e_harness/client_apps/codex/transcripts/fallback_metadata.jsonl @@ -0,0 +1,6 @@ +{"type":"thread.started","thread_id":"01a18d5f-e8d7-7c70-9059-c8a74922438e"} +{"type":"item.completed","item":{"id":"item_0","type":"error","message":"Model metadata for `gemini-4-argon-codex-compat-vertex-ai` not found. Defaulting to fallback metadata; this can degrade performance and cause issues."}} +{"type":"turn.started"} +{"type":"item.completed","item":{"id":"item_1","type":"reasoning","text":"**Enumerating Primes**\n\nI've successfully identified all prime numbers within the 1 to 50 range and meticulously counted each one, establishing their sequence and quantity.\n\n"}} +{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"15"}} +{"type":"turn.completed","usage":{"input_tokens":8517,"cached_input_tokens":0,"cache_write_input_tokens":0,"output_tokens":211,"reasoning_output_tokens":205}} diff --git a/tests/e2e_harness/client_apps/codex/transcripts/file_change.jsonl b/tests/e2e_harness/client_apps/codex/transcripts/file_change.jsonl new file mode 100644 index 00000000000..59027682dd2 --- /dev/null +++ b/tests/e2e_harness/client_apps/codex/transcripts/file_change.jsonl @@ -0,0 +1,7 @@ +{"type":"thread.started","thread_id":"01a18d5f-e9af-74e2-9631-f1abd29018b5"} +{"type":"turn.started"} +{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I’ll create `notes.txt` with the requested line."}} +{"type":"item.started","item":{"id":"item_1","type":"file_change","changes":[{"path":"/tmp/codex-session/workdir/notes.txt","kind":"add"}],"status":"in_progress"}} +{"type":"item.completed","item":{"id":"item_1","type":"file_change","changes":[{"path":"/tmp/codex-session/workdir/notes.txt","kind":"add"}],"status":"completed"}} +{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"Created `notes.txt` containing only `q7w8e9`."}} +{"type":"turn.completed","usage":{"input_tokens":21906,"cached_input_tokens":10882,"cache_write_input_tokens":11018,"output_tokens":131,"reasoning_output_tokens":0}} diff --git a/tests/e2e_harness/client_apps/codex/transcripts/first_turn.jsonl b/tests/e2e_harness/client_apps/codex/transcripts/first_turn.jsonl new file mode 100644 index 00000000000..34ad26641b5 --- /dev/null +++ b/tests/e2e_harness/client_apps/codex/transcripts/first_turn.jsonl @@ -0,0 +1,4 @@ +{"type":"thread.started","thread_id":"01a18d64-83eb-78f3-b6a3-5d7ae37d3d93"} +{"type":"turn.started"} +{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"OK"}} +{"type":"turn.completed","usage":{"input_tokens":10873,"cached_input_tokens":0,"cache_write_input_tokens":10870,"output_tokens":5,"reasoning_output_tokens":0}} diff --git a/tests/e2e_harness/client_apps/codex/transcripts/rate_limited.jsonl b/tests/e2e_harness/client_apps/codex/transcripts/rate_limited.jsonl new file mode 100644 index 00000000000..a0e922f81b2 --- /dev/null +++ b/tests/e2e_harness/client_apps/codex/transcripts/rate_limited.jsonl @@ -0,0 +1,9 @@ +{"type":"thread.started","thread_id":"01a18d84-ca4f-77b1-8367-f3420df1b7a5"} +{"type":"turn.started"} +{"type":"error","message":"Reconnecting... 1/5 (rate limit exceeded: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-6.1-sol for gpt-6.1-sol in eastus2 have exceeded token rate limit.)"} +{"type":"error","message":"Reconnecting... 2/5 (rate limit exceeded: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-6.1-sol for gpt-6.1-sol in eastus2 have exceeded token rate limit.)"} +{"type":"error","message":"Reconnecting... 3/5 (rate limit exceeded: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-6.1-sol for gpt-6.1-sol in eastus2 have exceeded token rate limit.)"} +{"type":"error","message":"Reconnecting... 4/5 (rate limit exceeded: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-6.1-sol for gpt-6.1-sol in eastus2 have exceeded token rate limit.)"} +{"type":"error","message":"Reconnecting... 5/5 (rate limit exceeded: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-6.1-sol for gpt-6.1-sol in eastus2 have exceeded token rate limit.)"} +{"type":"error","message":"rate limit exceeded: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-6.1-sol for gpt-6.1-sol in eastus2 have exceeded token rate limit."} +{"type":"turn.failed","error":{"message":"rate limit exceeded: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-6.1-sol for gpt-6.1-sol in eastus2 have exceeded token rate limit."}} diff --git a/tests/e2e_harness/client_apps/codex/transcripts/resumed_turn.jsonl b/tests/e2e_harness/client_apps/codex/transcripts/resumed_turn.jsonl new file mode 100644 index 00000000000..de793588de8 --- /dev/null +++ b/tests/e2e_harness/client_apps/codex/transcripts/resumed_turn.jsonl @@ -0,0 +1,4 @@ +{"type":"thread.started","thread_id":"01a18d64-83eb-78f3-b6a3-5d7ae37d3d93"} +{"type":"turn.started"} +{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"p3q4r5"}} +{"type":"turn.completed","usage":{"input_tokens":21772,"cached_input_tokens":10870,"cache_write_input_tokens":10896,"output_tokens":15,"reasoning_output_tokens":0}} diff --git a/tests/e2e_harness/client_apps/codex/transcripts/shell_command.jsonl b/tests/e2e_harness/client_apps/codex/transcripts/shell_command.jsonl new file mode 100644 index 00000000000..db1f7a7ef35 --- /dev/null +++ b/tests/e2e_harness/client_apps/codex/transcripts/shell_command.jsonl @@ -0,0 +1,6 @@ +{"type":"thread.started","thread_id":"01a18d86-cf06-7c42-8951-618e79c2013e"} +{"type":"turn.started"} +{"type":"item.started","item":{"id":"item_0","type":"command_execution","command":"/bin/bash -lc 'echo c887862269b3'","aggregated_output":"","exit_code":null,"status":"in_progress"}} +{"type":"item.completed","item":{"id":"item_0","type":"command_execution","command":"/bin/bash -lc 'echo c887862269b3'","aggregated_output":"c887862269b3\n","exit_code":0,"status":"completed"}} +{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"c887862269b3"}} +{"type":"turn.completed","usage":{"input_tokens":21984,"cached_input_tokens":10955,"cache_write_input_tokens":11023,"output_tokens":48,"reasoning_output_tokens":0}} diff --git a/tests/e2e_harness/client_apps/codex/transcripts/turn_failed.jsonl b/tests/e2e_harness/client_apps/codex/transcripts/turn_failed.jsonl new file mode 100644 index 00000000000..0b94c48af44 --- /dev/null +++ b/tests/e2e_harness/client_apps/codex/transcripts/turn_failed.jsonl @@ -0,0 +1,4 @@ +{"type":"thread.started","thread_id":"01a18d67-3419-7430-85ab-dd4795bfd01c"} +{"type":"turn.started"} +{"type":"error","message":"{\"error\":{\"message\":\"/responses: Invalid model name passed in model=gpt-6.1-sol-does-not-exist. Call `/v1/models` to view available models for your key.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"400\",\"provider_specific_fields\":{\"error\":\"/responses: Invalid model name passed in model=gpt-6.1-sol-does-not-exist. Call `/v1/models` to view available models for your key.\"}}}"} +{"type":"turn.failed","error":{"message":"{\"error\":{\"message\":\"/responses: Invalid model name passed in model=gpt-6.1-sol-does-not-exist. Call `/v1/models` to view available models for your key.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"400\",\"provider_specific_fields\":{\"error\":\"/responses: Invalid model name passed in model=gpt-6.1-sol-does-not-exist. Call `/v1/models` to view available models for your key.\"}}}"}} diff --git a/tests/e2e_harness/client_apps/codex/transcripts/web_search.jsonl b/tests/e2e_harness/client_apps/codex/transcripts/web_search.jsonl new file mode 100644 index 00000000000..5d99e4e252e --- /dev/null +++ b/tests/e2e_harness/client_apps/codex/transcripts/web_search.jsonl @@ -0,0 +1,7 @@ +{"type":"thread.started","thread_id":"01a18d67-686f-7e72-bc9d-f6a24e2320ed"} +{"type":"turn.started"} +{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I’ll look up LiteLLM’s documentation domain."}} +{"type":"item.started","item":{"id":"item_1","type":"web_search","id":"exec-baaa41bf-4055-42d4-8651-fbd2611cf820","query":"","action":{"type":"other"}}} +{"type":"item.completed","item":{"id":"item_1","type":"web_search","id":"exec-baaa41bf-4055-42d4-8651-fbd2611cf820","query":"LiteLLM documentation official","action":{"type":"search","query":"LiteLLM documentation official"}}} +{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"[docs.litellm.ai](https://docs.litellm.ai/)"}} +{"type":"turn.completed","usage":{"input_tokens":30618,"cached_input_tokens":13312,"cache_write_input_tokens":17300,"output_tokens":80,"reasoning_output_tokens":0}} diff --git a/tests/e2e_harness/client_apps/codex/transcripts/web_search_never_completed.jsonl b/tests/e2e_harness/client_apps/codex/transcripts/web_search_never_completed.jsonl new file mode 100644 index 00000000000..bb50e8d1e55 --- /dev/null +++ b/tests/e2e_harness/client_apps/codex/transcripts/web_search_never_completed.jsonl @@ -0,0 +1,7 @@ +{"type":"thread.started","thread_id":"01a18d65-9cd0-7cf0-8fdd-ea480a832271"} +{"type":"turn.started"} +{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I’ll look up LiteLLM’s documentation domain."}} +{"type":"item.started","item":{"id":"item_1","type":"web_search","id":"exec-dfea1d72-2e16-4853-a9c5-713fbd2d4a45","query":"","action":{"type":"other"}}} +{"type":"item.started","item":{"id":"item_2","type":"web_search","id":"exec-f0f475b4-d3a1-440d-988a-cad646913023","query":"","action":{"type":"other"}}} +{"type":"item.completed","item":{"id":"item_3","type":"agent_message","text":"docs.litellm.ai"}} +{"type":"turn.completed","usage":{"input_tokens":59995,"cached_input_tokens":43336,"cache_write_input_tokens":16647,"output_tokens":154,"reasoning_output_tokens":14}} diff --git a/tests/e2e_harness/client_apps/test_compat_collector.py b/tests/e2e_harness/client_apps/test_compat_collector.py new file mode 100644 index 00000000000..76bcd4787a1 --- /dev/null +++ b/tests/e2e_harness/client_apps/test_compat_collector.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Final, Literal + +import pytest + +from client_apps.compat_collector import NO_RESULT_ERROR, CellAddress, cell_address, clear_shards, outcome_rows, publish +from client_apps.compat_results import Failed, NotApplicable, Passed, ResultRow, read_rows, shard_dir + +MANIFEST: Final = """schema_version: "1" +client: fakeclient +package: fake-client +providers: [openai, vertex_ai] +features: + - {id: vision, name: Vision} +""" +ADDRESS: Final = CellAddress(client="fakeclient", feature_id="vision", provider="openai") +RESULTS_ENV: Final = "FAKECLIENT_COMPAT_RESULTS_PATH" + + +def _clients_dir(root: Path) -> Path: + client: Final = root / "fakeclient" + client.mkdir(parents=True) + _ = (client / "manifest.yaml").write_text(MANIFEST) + return root + + +def _report( + when: Literal["setup", "call", "teardown"], outcome: Literal["passed", "failed"], longrepr: str | None = None +) -> pytest.TestReport: + return pytest.TestReport( + nodeid="fakeclient/vision/test_openai.py::test_vision_openai", + location=("fakeclient/vision/test_openai.py", 1, "test_vision_openai"), + keywords={}, + outcome=outcome, + longrepr=longrepr, + when=when, + ) + + +def _row(model: str | None, outcome: Passed | Failed | NotApplicable) -> ResultRow: + return ResultRow(feature_id="vision", provider="openai", model=model, outcome=outcome) + + +@pytest.mark.parametrize( + ("relative", "expected"), + ( + ("fakeclient/vision/test_openai.py", ADDRESS), + ("fakeclient/vision/test_vertex_ai.py", CellAddress("fakeclient", "vision", "vertex_ai")), + ("fakeclient/vision/test_azure.py", None), + ("fakeclient/web_search/test_openai.py", None), + ("fakeclient/vision/helpers.py", None), + ("fakeclient/test_openai.py", None), + ("otherclient/vision/test_openai.py", None), + ), +) +def test_cell_address_only_maps_manifest_cells(tmp_path: Path, relative: str, expected: CellAddress | None) -> None: + clients: Final = _clients_dir(tmp_path) + + assert cell_address(clients / relative, clients) == expected + + +def test_cell_address_ignores_files_outside_the_clients_dir(tmp_path: Path) -> None: + clients: Final = _clients_dir(tmp_path / "clients") + + assert cell_address(tmp_path / "fakeclient" / "vision" / "test_openai.py", clients) is None + + +def test_a_cell_that_passed_without_reporting_is_recorded_as_failed() -> None: + assert outcome_rows(ADDRESS, (), _report("call", "passed")) == (_row(None, Failed(error=NO_RESULT_ERROR)),) + + +def test_a_cell_that_crashed_records_the_assertion_line() -> None: + longrepr: Final = "def test():\n> assert x\nE AssertionError: proxy answered 500\n\nfile.py:3: AssertionError" + + assert outcome_rows(ADDRESS, (), _report("call", "failed", longrepr)) == ( + _row(None, Failed(error="AssertionError: proxy answered 500")), + ) + + +def test_a_setup_failure_is_a_failed_cell() -> None: + rows: Final = outcome_rows(ADDRESS, (), _report("setup", "failed", "could not register gpt-6.1-sol")) + + assert rows == (_row(None, Failed(error="could not register gpt-6.1-sol")),) + + +@pytest.mark.parametrize( + ("recorded", "report"), + ( + ((_row("m", Failed(error="no file_change")),), _report("call", "failed", "Failed: m: no file_change")), + ((_row("m", Passed()),), _report("call", "passed")), + ((_row(None, NotApplicable(reason="no apply_patch")),), _report("call", "passed")), + ((), _report("teardown", "failed", "delete_model warned")), + ((), _report("setup", "passed")), + ), +) +def test_a_cell_that_reported_its_own_outcome_gets_no_extra_row( + recorded: tuple[ResultRow, ...], report: pytest.TestReport +) -> None: + assert outcome_rows(ADDRESS, recorded, report) == () + + +def test_workers_write_shards_and_the_controller_merges_them(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + results: Final = tmp_path / "out" / "fakeclient.json" + monkeypatch.setenv(RESULTS_ENV, str(results)) + first: Final = _row("a", Passed()) + second: Final = _row("b", Failed(error="boom")) + + publish("fakeclient", (second,), "gw1") + publish("fakeclient", (first,), "gw0") + publish("fakeclient", (), "gw2") + + assert sorted(path.name for path in shard_dir(results).iterdir()) == ["gw0.json", "gw1.json"] + assert not results.exists() + + publish("fakeclient", (), None) + + assert read_rows(results) == (first, second) + + +def test_clear_shards_drops_a_previous_runs_rows(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + results: Final = tmp_path / "fakeclient.json" + monkeypatch.setenv(RESULTS_ENV, str(results)) + publish("fakeclient", (_row("stale", Passed()),), "gw0") + + clear_shards("fakeclient") + publish("fakeclient", (_row("fresh", Passed()),), None) + + assert [row.model for row in read_rows(results)] == ["fresh"] + + +def test_a_run_with_no_cells_leaves_the_results_file_alone(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + results: Final = tmp_path / "fakeclient.json" + monkeypatch.setenv(RESULTS_ENV, str(results)) + _ = results.write_text("[]") + + publish("fakeclient", (), None) + + assert results.read_text() == "[]" diff --git a/tests/e2e_harness/client_apps/test_compat_matrix_builder.py b/tests/e2e_harness/client_apps/test_compat_matrix_builder.py new file mode 100644 index 00000000000..13b199ead8b --- /dev/null +++ b/tests/e2e_harness/client_apps/test_compat_matrix_builder.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Final + +import pytest + +from client_apps.compat_results import Failed, NotApplicable, NotTested, Outcome, Passed, ResultRow, write_rows +from client_apps.matrix_builder import ( + REGRESSION_EXIT_CODE, + REGRESSIONS, + Manifest, + ManifestFeature, + Matrix, + Regression, + aggregate, + build_matrix, + find_regressions, + load_manifest, + main, + pinned_client_version, +) + +MANIFEST: Final = Manifest( + schema_version="1", + client="codex", + package="@openai/codex", + providers=("openai", "vertex_ai"), + features=(ManifestFeature(id="vision", name="Vision"), ManifestFeature(id="web_search", name="Web search")), +) +GENERATED_AT: Final = datetime(2026, 1, 2, tzinfo=timezone.utc) + + +def _row(model: str | None, outcome: Outcome, feature_id: str = "vision", provider: str = "openai") -> ResultRow: + return ResultRow(feature_id=feature_id, provider=provider, model=model, outcome=outcome) + + +def _matrix(rows: tuple[ResultRow, ...]) -> Matrix: + return build_matrix(MANIFEST, rows, litellm_version="1.0.0", client_version="0.1.0", generated_at=GENERATED_AT) + + +@pytest.mark.parametrize( + ("rows", "expected"), + ( + ((_row("a", Passed()), _row("b", Passed())), Passed()), + ((_row("a", Passed()), _row("b", Failed(error="no image"))), Failed(error="b: no image")), + ( + (_row("a", Failed(error="x")), _row(None, Failed(error="setup broke"))), + Failed(error="a: x; setup broke"), + ), + ((_row("a", Passed()), _row("b", NotTested())), Passed()), + ((_row(None, NotApplicable(reason="no apply_patch")),), NotApplicable(reason="no apply_patch")), + ((), NotTested()), + ), +) +def test_a_cell_is_green_only_when_no_model_failed(rows: tuple[ResultRow, ...], expected: Outcome) -> None: + assert aggregate(rows) == expected + + +def test_build_matrix_fills_every_manifest_cell_and_drops_unknown_rows() -> None: + matrix: Final = _matrix( + ( + _row("a", Passed()), + _row("a", Failed(error="404"), feature_id="web_search", provider="vertex_ai"), + _row("a", Passed(), feature_id="retired_feature"), + _row("a", Passed(), provider="retired_provider"), + ) + ) + + assert [(feature.id, dict(feature.providers)) for feature in matrix.features] == [ + ("vision", {"openai": Passed(), "vertex_ai": NotTested()}), + ("web_search", {"openai": NotTested(), "vertex_ai": Failed(error="a: 404")}), + ] + assert matrix.generated_at == GENERATED_AT.isoformat() + + +def test_find_regressions_flags_only_green_to_red() -> None: + old: Final = _matrix( + ( + _row("a", Passed()), + _row("a", Failed(error="old"), provider="vertex_ai"), + _row("a", Passed(), feature_id="web_search"), + ) + ) + new: Final = _matrix( + ( + _row("a", Failed(error="429")), + _row("a", Failed(error="still red"), provider="vertex_ai"), + _row("a", Failed(error="new cell"), feature_id="web_search", provider="vertex_ai"), + _row("a", Passed(), feature_id="web_search"), + ) + ) + + assert find_regressions(old, new) == ( + Regression(feature_id="vision", feature_name="Vision", provider="openai", error="a: 429"), + ) + + +def test_matrix_round_trips_through_json() -> None: + matrix: Final = _matrix((_row("a", Passed()), _row(None, NotApplicable(reason="n/a"), provider="vertex_ai"))) + + assert Matrix.model_validate_json(matrix.model_dump_json()) == matrix + + +def test_cli_build_then_regressions_exit_codes(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + manifest: Final = tmp_path / "manifest.yaml" + _ = manifest.write_text(json.dumps(MANIFEST.model_dump(mode="json"))) + green_rows: Final = tmp_path / "green.json" + red_rows: Final = tmp_path / "red.json" + write_rows(green_rows, (_row("a", Passed()),)) + write_rows(red_rows, (_row("a", Failed(error="400 from proxy")),)) + green: Final = tmp_path / "green-matrix.json" + red: Final = tmp_path / "red-matrix.json" + build: Final = ("build", "--manifest", str(manifest), "--litellm-version", "9.9.9") + + assert main((*build, "--results", str(green_rows), "--out", str(green))) == 0 + assert main((*build, "--results", str(red_rows), "--out", str(red))) == 0 + built: Final = Matrix.model_validate_json(red.read_bytes()) + assert (built.litellm_version, built.client_version) == ("9.9.9", pinned_client_version("@openai/codex")) + _ = capsys.readouterr() + + assert main(("regressions", "--old", str(green), "--new", str(red))) == REGRESSION_EXIT_CODE + regressions: Final = REGRESSIONS.validate_json(capsys.readouterr().out) + assert [regression.error for regression in regressions] == ["a: 400 from proxy"] + assert main(("regressions", "--old", str(red), "--new", str(green))) == 0 + assert main(("regressions", "--old", str(tmp_path / "missing.json"), "--new", str(red))) == 0 + + +def test_shipped_manifests_validate_and_pin_their_client() -> None: + manifests: Final = tuple(Path(__file__).resolve().parents[2].glob("e2e/client_apps/*/manifest.yaml")) + + assert manifests + for path in manifests: + manifest = load_manifest(path) + assert len({feature.id for feature in manifest.features}) == len(manifest.features) + assert pinned_client_version(manifest.package)