From 014edcbb00d5ab823cb2ea931876d3bf0ecd4470 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:09:37 -0700 Subject: [PATCH] test(e2e): mark known proxy gaps in the Codex matrix as strict xfails The codex manifest gets a known_gaps list of feature, provider and reason. The client_apps conftest marks each listed cell xfail(strict=True) at collection, so the shared e2e lane stays green while a known gap lasts and goes red when the cell starts passing. The results file still records what the cell did, including crashes under the xfail and a passing strict XPASS. An entry naming a feature or provider the manifest lacks, or a cell listed twice, fails loudly. Web search is listed for all four columns with the proxy behavior behind each failure. --- tests/e2e/AGENTS.md | 2 +- tests/e2e/client_apps/codex/manifest.yaml | 18 +++ tests/e2e/client_apps/compat_collector.py | 42 +++-- tests/e2e/client_apps/conftest.py | 21 ++- tests/e2e/client_apps/matrix_builder.py | 36 ++++- tests/e2e_harness/AGENTS.md | 2 +- .../client_apps/test_compat_collector.py | 145 ++++++++++++++++-- .../client_apps/test_compat_matrix_builder.py | 58 +++++++ 8 files changed, 288 insertions(+), 36 deletions(-) diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index be7a1240fbc..16d29877668 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -49,7 +49,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `secret_manager/` - the gateway's `key_management_system` against a real secret manager: deployment keys resolved from it (`os.environ/` where the name exists only in the manager) and virtual keys written to and deleted from it. The tests are backend-agnostic and each backend is its own lane, because the setting is global to the proxy: `E2E_SECRET_MANAGER=` opts in and picks the backend from `secret_backends.BACKENDS`, the proxy is booted from `gateway/secret_manager__ci_config.yml` against the live manager, and the tests reach that manager through the backend's `SecretStore` (`secret_store_.py`). A test needing something not every backend does carries `requires_capability(...)` and is deselected on lanes that lack it. `secret_manager/backend.sh up ` runs a backend in Docker and writes the proxy's and the tests' env. Marked `secret_manager`, deselected unless `E2E_SECRET_MANAGER` is set, and kept out of the per-PR selector. Backends today: `hashicorp_vault` and `cyberark` (CyberArk Conjur, which cannot delete, so the delete test is Vault-only) - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke -- `client_apps/` - compatibility matrices for coding-agent CLIs other than Claude Code, one subfolder per client (`codex/` today). A client folder holds `manifest.yaml` (its provider columns and feature rows), `proxy_models.yaml` (the deployments each column registers through `/model/new` on first use and deletes at session end) and one cell per `/test_.py`. A cell drives the real CLI against the proxy with a scoped key for every model in its column and reports one tagged-union outcome per model through the `compat_result` fixture, and the matrix marks the cell green only when every model passed. Cells of one column take turns across xdist workers through a file lock, since a provider's rate limit is shared by the whole column, while different columns run in parallel. The CLIs are pinned in `client_apps/package.json` and its lockfile, the runner image installs them with `npm ci`, and the driver runs `node_modules/.bin/` before falling back to the one on `PATH`. Nothing here imports `claude_code/`. To run it locally, run `npm ci` in `tests/e2e/client_apps`, then `uv run pytest tests/e2e/client_apps -n 8` with `LITELLM_PROXY_URL` and `LITELLM_MASTER_KEY` set. Point pytest at `tests/e2e/client_apps` or below, because `client_apps/conftest.py` is what writes the results to `_COMPAT_RESULTS_PATH` (default `-compat-results.json`, merged across xdist workers). `cd tests/e2e && python -m client_apps.matrix_builder build --manifest client_apps/codex/manifest.yaml --results --out ` turns them into the matrix, and `regressions --old --new ` exits 3 when a green cell turned red. The builder, collector, parser and driver are covered by `tests/e2e_harness/client_apps/` +- `client_apps/` - compatibility matrices for coding-agent CLIs other than Claude Code, one subfolder per client (`codex/` today). A client folder holds `manifest.yaml` (its provider columns and feature rows), `proxy_models.yaml` (the deployments each column registers through `/model/new` on first use and deletes at session end) and one cell per `/test_.py`. A cell drives the real CLI against the proxy with a scoped key for every model in its column and reports one tagged-union outcome per model through the `compat_result` fixture, and the matrix marks the cell green only when every model passed. A proxy gap we already know about goes in the manifest's `known_gaps` as `{feature, provider, reason}`, the reason naming the proxy behavior in plain words, and `client_apps/conftest.py` marks that cell `xfail(strict=True)`, so the lane stays green while the gap lasts and goes red when the cell starts passing and the entry should go. The results file still records what the cell really did, and an entry naming a feature or provider the manifest lacks stops the run at collection. Cells of one column take turns across xdist workers through a file lock, since a provider's rate limit is shared by the whole column, while different columns run in parallel. The CLIs are pinned in `client_apps/package.json` and its lockfile, the runner image installs them with `npm ci`, and the driver runs `node_modules/.bin/` before falling back to the one on `PATH`. Nothing here imports `claude_code/`. To run it locally, run `npm ci` in `tests/e2e/client_apps`, then `uv run pytest tests/e2e/client_apps -n 8` with `LITELLM_PROXY_URL` and `LITELLM_MASTER_KEY` set. Point pytest at `tests/e2e/client_apps` or below, because `client_apps/conftest.py` is what writes the results to `_COMPAT_RESULTS_PATH` (default `-compat-results.json`, merged across xdist workers). `cd tests/e2e && python -m client_apps.matrix_builder build --manifest client_apps/codex/manifest.yaml --results --out ` turns them into the matrix, and `regressions --old --new ` exits 3 when a green cell turned red. The builder, collector, parser and driver are covered by `tests/e2e_harness/client_apps/` - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream, see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport. The Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`. Its specs follow the dashboard's rule against testing what shadcn and Base UI already guarantee, in `ui/litellm-dashboard/AGENTS.md` ## MCP suite: real Datadog only diff --git a/tests/e2e/client_apps/codex/manifest.yaml b/tests/e2e/client_apps/codex/manifest.yaml index 626356c16d7..4328f8af0d8 100644 --- a/tests/e2e/client_apps/codex/manifest.yaml +++ b/tests/e2e/client_apps/codex/manifest.yaml @@ -27,3 +27,21 @@ features: name: Structured output - id: web_search name: Web search + +known_gaps: + - feature: web_search + provider: openai + reason: The proxy has no POST /v1/alpha/search route, so Codex's web search calls get a 404 and never complete + - feature: web_search + provider: azure + reason: >- + The proxy has no POST /v1/alpha/search route, and the Responses WebSocket relay forwards Codex's OpenAI-only + internal_chat_message_metadata_passthrough input field, which Azure rejects with a 400 + - feature: web_search + provider: bedrock_mantle + reason: >- + The Responses WebSocket replay drops Codex's custom_tool_call output item, so Bedrock rejects the next turn with + a 400, and the proxy's error frame has no status, so Codex waits until the timeout + - feature: web_search + provider: vertex_ai + reason: The proxy drops Gemini's search tool whenever the request also has function tools, which Codex always sends diff --git a/tests/e2e/client_apps/compat_collector.py b/tests/e2e/client_apps/compat_collector.py index b4af6aa67d7..2d504223bdc 100644 --- a/tests/e2e/client_apps/compat_collector.py +++ b/tests/e2e/client_apps/compat_collector.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +from collections.abc import Iterable from dataclasses import dataclass from functools import cache from itertools import chain @@ -39,7 +40,7 @@ def client_manifest(client_dir: Path) -> Manifest: return load_manifest(client_dir / "manifest.yaml") -def cell_address(test_file: Path, clients_dir: Path = CLIENT_APPS_DIR) -> CellAddress | None: +def cell_address(test_file: Path, clients_dir: Path) -> CellAddress | None: resolved: Final = test_file.resolve() if not resolved.is_relative_to(clients_dir): return None @@ -85,16 +86,33 @@ class CompatRecorder: _record(self._item, _row(self.address, None, NotApplicable(reason=reason))) -def _fails_cell(report: pytest.TestReport) -> bool: - return report.failed and report.when in ("setup", "call") +def known_gap_marks( + items: Iterable[pytest.Item], clients_dir: Path +) -> tuple[tuple[pytest.Item, pytest.MarkDecorator], ...]: + marked: Final = ((item, _known_gap_mark(item.path, clients_dir)) for item in items) + return tuple((item, mark) for item, mark in marked if mark is not None) + + +def _known_gap_mark(test_file: Path, clients_dir: Path) -> pytest.MarkDecorator | None: + address: Final = cell_address(test_file, clients_dir) + if address is None: + return None + gap: Final = client_manifest(clients_dir / address.client).known_gap(address.feature_id, address.provider) + return None if gap is None else pytest.mark.xfail(strict=True, reason=gap.reason) + + +def _raised(call: pytest.CallInfo[None]) -> bool: + return call.excinfo is not None and not call.excinfo.errisinstance(pytest.skip.Exception) def outcome_rows( - address: CellAddress, recorded: tuple[ResultRow, ...], report: pytest.TestReport + address: CellAddress, recorded: tuple[ResultRow, ...], call: pytest.CallInfo[None], report: pytest.TestReport ) -> tuple[ResultRow, ...]: - if _fails_cell(report) and not any(isinstance(row.outcome, Failed) for row in recorded): + if call.when not in ("setup", "call"): + return () + if _raised(call) and not any(isinstance(row.outcome, Failed) for row in recorded): return (_row(address, None, Failed(error=_crash_message(report))),) - if report.passed and report.when == "call" and not recorded: + if call.when == "call" and call.excinfo is None and not recorded: return (_row(address, None, Failed(error=NO_RESULT_ERROR)),) return () @@ -105,21 +123,21 @@ def _crash_message(report: pytest.TestReport) -> str: return (error_lines or lines or (f"{report.when} failed",))[-1].removeprefix("E ").strip() -def known_clients() -> tuple[str, ...]: - return tuple(sorted(manifest.parent.name for manifest in CLIENT_APPS_DIR.glob("*/manifest.yaml"))) +def known_clients(clients_dir: Path) -> tuple[str, ...]: + return tuple(sorted(manifest.parent.name for manifest in clients_dir.glob("*/manifest.yaml"))) def worker_id() -> str | None: return os.environ.get("PYTEST_XDIST_WORKER") -def _item_rows(item: pytest.Item, client: str) -> tuple[ResultRow, ...]: - address: Final = cell_address(item.path) +def _item_rows(item: pytest.Item, client: str, clients_dir: Path) -> tuple[ResultRow, ...]: + address: Final = cell_address(item.path, clients_dir) return item.stash.get(ROWS, ()) if address is not None and address.client == client else () -def client_rows(session: pytest.Session, client: str) -> tuple[ResultRow, ...]: - return tuple(chain.from_iterable(_item_rows(item, client) for item in session.items)) +def client_rows(session: pytest.Session, client: str, clients_dir: Path) -> tuple[ResultRow, ...]: + return tuple(chain.from_iterable(_item_rows(item, client, clients_dir) for item in session.items)) def clear_shards(client: str) -> None: diff --git a/tests/e2e/client_apps/conftest.py b/tests/e2e/client_apps/conftest.py index 1c8d6233d05..97cd2cb21a1 100644 --- a/tests/e2e/client_apps/conftest.py +++ b/tests/e2e/client_apps/conftest.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Generator +from pathlib import Path from typing import Final import pytest @@ -12,17 +13,25 @@ from client_apps.compat_collector import ( clear_shards, client_rows, known_clients, + known_gap_marks, outcome_rows, publish, worker_id, ) +CLIENTS_DIR: Final = Path(__file__).resolve().parent + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + for item, mark in known_gap_marks(items, CLIENTS_DIR): + item.add_marker(mark) + @pytest.fixture def compat_result(request: pytest.FixtureRequest) -> CompatRecorder: item: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untyped assert isinstance(item, pytest.Item) - address: Final = cell_address(item.path) + address: Final = cell_address(item.path, CLIENTS_DIR) if address is None: pytest.fail(f"{item.path} is not a //test_.py cell of a client manifest") return CompatRecorder(item, address) @@ -38,10 +47,10 @@ def pytest_runtest_makereport( item: pytest.Item, call: pytest.CallInfo[None] ) -> Generator[None, pytest.TestReport, pytest.TestReport]: report: Final = yield - address: Final = cell_address(item.path) + address: Final = cell_address(item.path, CLIENTS_DIR) if address is None: return report - extra: Final = outcome_rows(address, item.stash.get(ROWS, ()), report) + extra: Final = outcome_rows(address, item.stash.get(ROWS, ()), call, report) if extra: item.stash[ROWS] = (*item.stash.get(ROWS, ()), *extra) return report @@ -50,10 +59,10 @@ def pytest_runtest_makereport( def pytest_sessionstart(session: pytest.Session) -> None: if worker_id() is not None: return - for client in known_clients(): + for client in known_clients(CLIENTS_DIR): clear_shards(client) def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: - for client in known_clients(): - publish(client, client_rows(session, client), worker_id()) + for client in known_clients(CLIENTS_DIR): + publish(client, client_rows(session, client, CLIENTS_DIR), worker_id()) diff --git a/tests/e2e/client_apps/matrix_builder.py b/tests/e2e/client_apps/matrix_builder.py index f50a404f351..0adea3320ef 100644 --- a/tests/e2e/client_apps/matrix_builder.py +++ b/tests/e2e/client_apps/matrix_builder.py @@ -5,12 +5,13 @@ from argparse import ArgumentParser from collections.abc import Iterator, Mapping from datetime import datetime, timezone from importlib.metadata import version +from itertools import chain from pathlib import Path from types import MappingProxyType -from typing import Annotated, Final, Literal +from typing import Annotated, Final, Literal, Self import yaml -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, model_validator from client_apps.compat_results import ( Failed, @@ -35,12 +36,43 @@ class ManifestFeature(_Frozen): name: str +class KnownGap(_Frozen): + feature: str + provider: str + reason: str = Field(min_length=1) + + class Manifest(_Frozen): schema_version: str client: str package: str providers: tuple[str, ...] features: tuple[ManifestFeature, ...] + known_gaps: tuple[KnownGap, ...] = () + + @model_validator(mode="after") + def _known_gaps_name_manifest_cells(self) -> Self: + problems: Final = tuple(chain.from_iterable(self._gap_problems(gap) for gap in self.known_gaps)) + if problems: + raise ValueError(f"known_gaps: {'; '.join(problems)}") + cells: Final = tuple(f"{gap.feature}/{gap.provider}" for gap in self.known_gaps) + repeated: Final = sorted(frozenset(cell for cell in cells if cells.count(cell) > 1)) + if repeated: + raise ValueError(f"known_gaps lists a cell more than once: {', '.join(repeated)}") + return self + + def _gap_problems(self, gap: KnownGap) -> tuple[str, ...]: + feature_problem: Final = ( + () if any(feature.id == gap.feature for feature in self.features) else (f"unknown feature {gap.feature!r}",) + ) + provider_problem: Final = () if gap.provider in self.providers else (f"unknown provider {gap.provider!r}",) + return (*feature_problem, *provider_problem) + + def known_gap(self, feature_id: str, provider: str) -> KnownGap | None: + return next( + (gap for gap in self.known_gaps if gap.feature == feature_id and gap.provider == provider), + None, + ) class MatrixFeature(_Frozen): diff --git a/tests/e2e_harness/AGENTS.md b/tests/e2e_harness/AGENTS.md index 9bfa2898095..d1dd4f21837 100644 --- a/tests/e2e_harness/AGENTS.md +++ b/tests/e2e_harness/AGENTS.md @@ -2,7 +2,7 @@ Tests of the harness under `tests/e2e/` (the transport, the clients, the fixture bundle and replay edge, the stack lock, the IdP launcher, the coverage collector, the JUnit properties, the Claude Code driver and the client compatibility matrices), not of the product. They live outside `tests/e2e/` because the Buildkite e2e run copies that folder into the runner image and runs every test in it, so a harness test in there counts as a product test in the nightly numbers. Nothing here needs a proxy, provider keys or the network -The layout mirrors `tests/e2e/`: `test_e2e_http.py` covers `tests/e2e/e2e_http.py`, `logging/test_datadog_reader.py` covers `tests/e2e/logging/datadog_reader.py`, `claude_code/` covers the driver, builder, probe and version resolver, and `client_apps/` covers the client compatibility matrices' collector and builder, with `client_apps/codex/` covering the Codex event parser (against `codex exec --json` transcripts recorded from live runs), the CLI driver and the column lock. Put a new harness test under the folder that mirrors the suite folder whose module it covers +The layout mirrors `tests/e2e/`: `test_e2e_http.py` covers `tests/e2e/e2e_http.py`, `logging/test_datadog_reader.py` covers `tests/e2e/logging/datadog_reader.py`, `claude_code/` covers the driver, builder, probe and version resolver, and `client_apps/` covers the client compatibility matrices' collector and builder (running the real `client_apps/conftest.py` against a fake client through `pytester` to prove the known-gap xfail), with `client_apps/codex/` covering the Codex event parser (against `codex exec --json` transcripts recorded from live runs), the CLI driver and the column lock. Put a new harness test under the folder that mirrors the suite folder whose module it covers Run them from the repo root. `e2e_config` reads `LITELLM_MASTER_KEY` at import and any value will do, the CI lane sets a dummy: diff --git a/tests/e2e_harness/client_apps/test_compat_collector.py b/tests/e2e_harness/client_apps/test_compat_collector.py index 76bcd4787a1..332a3598a12 100644 --- a/tests/e2e_harness/client_apps/test_compat_collector.py +++ b/tests/e2e_harness/client_apps/test_compat_collector.py @@ -1,12 +1,24 @@ from __future__ import annotations +from datetime import datetime, timezone from pathlib import Path from typing import Final, Literal import pytest -from client_apps.compat_collector import NO_RESULT_ERROR, CellAddress, cell_address, clear_shards, outcome_rows, publish -from client_apps.compat_results import Failed, NotApplicable, Passed, ResultRow, read_rows, shard_dir +from client_apps.compat_collector import ( + CLIENT_APPS_DIR, + NO_RESULT_ERROR, + CellAddress, + cell_address, + clear_shards, + outcome_rows, + publish, +) +from client_apps.compat_results import Failed, NotApplicable, Outcome, Passed, ResultRow, read_rows, shard_dir +from client_apps.matrix_builder import build_matrix, load_manifest + +pytest_plugins = ("pytester",) MANIFEST: Final = """schema_version: "1" client: fakeclient @@ -27,7 +39,9 @@ def _clients_dir(root: Path) -> Path: def _report( - when: Literal["setup", "call", "teardown"], outcome: Literal["passed", "failed"], longrepr: str | None = None + when: Literal["setup", "call", "teardown"], + outcome: Literal["passed", "failed", "skipped"], + longrepr: str | None = None, ) -> pytest.TestReport: return pytest.TestReport( nodeid="fakeclient/vision/test_openai.py::test_vision_openai", @@ -67,38 +81,56 @@ def test_cell_address_ignores_files_outside_the_clients_dir(tmp_path: Path) -> N assert cell_address(tmp_path / "fakeclient" / "vision" / "test_openai.py", clients) is None +def _call(when: Literal["setup", "call", "teardown"], error: BaseException | None = None) -> pytest.CallInfo[None]: + def run() -> None: + if error is not None: + raise error + + return pytest.CallInfo[None].from_call(run, when) + + def test_a_cell_that_passed_without_reporting_is_recorded_as_failed() -> None: - assert outcome_rows(ADDRESS, (), _report("call", "passed")) == (_row(None, Failed(error=NO_RESULT_ERROR)),) + rows: Final = outcome_rows(ADDRESS, (), _call("call"), _report("call", "passed")) + + assert rows == (_row(None, Failed(error=NO_RESULT_ERROR)),) def test_a_cell_that_crashed_records_the_assertion_line() -> None: longrepr: Final = "def test():\n> assert x\nE AssertionError: proxy answered 500\n\nfile.py:3: AssertionError" + call: Final = _call("call", AssertionError("proxy answered 500")) - assert outcome_rows(ADDRESS, (), _report("call", "failed", longrepr)) == ( + assert outcome_rows(ADDRESS, (), call, _report("call", "failed", longrepr)) == ( _row(None, Failed(error="AssertionError: proxy answered 500")), ) def test_a_setup_failure_is_a_failed_cell() -> None: - rows: Final = outcome_rows(ADDRESS, (), _report("setup", "failed", "could not register gpt-6.1-sol")) + call: Final = _call("setup", RuntimeError("could not register gpt-6.1-sol")) + + rows: Final = outcome_rows(ADDRESS, (), call, _report("setup", "failed", "could not register gpt-6.1-sol")) assert rows == (_row(None, Failed(error="could not register gpt-6.1-sol")),) @pytest.mark.parametrize( - ("recorded", "report"), + ("recorded", "call", "report"), ( - ((_row("m", Failed(error="no file_change")),), _report("call", "failed", "Failed: m: no file_change")), - ((_row("m", Passed()),), _report("call", "passed")), - ((_row(None, NotApplicable(reason="no apply_patch")),), _report("call", "passed")), - ((), _report("teardown", "failed", "delete_model warned")), - ((), _report("setup", "passed")), + ( + (_row("m", Failed(error="no file_change")),), + _call("call", AssertionError("m: no file_change")), + _report("call", "failed", "Failed: m: no file_change"), + ), + ((_row("m", Passed()),), _call("call"), _report("call", "passed")), + ((_row(None, NotApplicable(reason="no apply_patch")),), _call("call"), _report("call", "passed")), + ((), _call("teardown", RuntimeError("delete_model warned")), _report("teardown", "failed", "delete_model")), + ((), _call("setup"), _report("setup", "passed")), + ((), _call("setup", pytest.skip.Exception("no codex binary")), _report("setup", "skipped")), ), ) def test_a_cell_that_reported_its_own_outcome_gets_no_extra_row( - recorded: tuple[ResultRow, ...], report: pytest.TestReport + recorded: tuple[ResultRow, ...], call: pytest.CallInfo[None], report: pytest.TestReport ) -> None: - assert outcome_rows(ADDRESS, recorded, report) == () + assert outcome_rows(ADDRESS, recorded, call, report) == () def test_workers_write_shards_and_the_controller_merges_them(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -138,3 +170,88 @@ def test_a_run_with_no_cells_leaves_the_results_file_alone(tmp_path: Path, monke publish("fakeclient", (), None) assert results.read_text() == "[]" + + +GAP_MANIFEST: Final = """schema_version: "1" +client: fakeclient +package: fake-client +providers: [openai, vertex_ai] +features: + - {id: vision, name: Vision} + - {id: web_search, name: Web search} +known_gaps: + - {feature: web_search, provider: openai, reason: the proxy has no search route} + - {feature: web_search, provider: vertex_ai, reason: the proxy drops search tools} + - {feature: vision, provider: vertex_ai, reason: the relay forwards an unknown field} +""" +CELL_BODIES: Final = ( + ( + "web_search", + "openai", + 'compat_result.failed("m", "no completed web_search item")\n' + ' pytest.fail("m: no completed web_search item", pytrace=False)', + ), + ("web_search", "vertex_ai", 'compat_result.passed("m")'), + ("vision", "openai", 'compat_result.passed("m")'), + ("vision", "vertex_ai", 'raise RuntimeError("proxy unreachable")'), +) + + +def _client_with_cells(pytester: pytest.Pytester, manifest: str) -> Path: + _ = pytester.makeconftest((CLIENT_APPS_DIR / "conftest.py").read_text()) + client: Final = pytester.path / "fakeclient" + client.mkdir() + _ = (client / "manifest.yaml").write_text(manifest) + for feature_id, provider, body in CELL_BODIES: + cell = client / feature_id / f"test_{provider}.py" + cell.parent.mkdir(exist_ok=True) + _ = cell.write_text(f"import pytest\n\n\ndef test_cell(compat_result):\n {body}\n") + return client + + +def _run_cells( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, manifest: str +) -> tuple[pytest.RunResult, Path]: + results: Final = pytester.path / "fakeclient-results.json" + monkeypatch.setenv(RESULTS_ENV, str(results)) + monkeypatch.delenv("PYTEST_XDIST_WORKER", raising=False) + client: Final = _client_with_cells(pytester, manifest) + return pytester.runpytest_inprocess("--import-mode=importlib", "-p", "no:cacheprovider", str(client)), results + + +def test_known_gap_cells_xfail_strictly_while_the_matrix_records_what_happened( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +) -> None: + run, results = _run_cells(pytester, monkeypatch, GAP_MANIFEST) + manifest: Final = load_manifest(pytester.path / "fakeclient" / "manifest.yaml") + matrix: Final = build_matrix( + manifest, + read_rows(results), + litellm_version="1.0.0", + client_version="0.1.0", + generated_at=datetime(2026, 1, 2, tzinfo=timezone.utc), + ) + cells: Final[dict[tuple[str, str], Outcome]] = { + (feature.id, provider): outcome for feature in matrix.features for provider, outcome in feature.providers.items() + } + + run.assert_outcomes(passed=1, failed=1, xfailed=2) + assert "the proxy drops search tools" in run.stdout.str() + assert cells == { + ("vision", "openai"): Passed(), + ("vision", "vertex_ai"): Failed(error="RuntimeError: proxy unreachable"), + ("web_search", "openai"): Failed(error="m: no completed web_search item"), + ("web_search", "vertex_ai"): Passed(), + } + + +def test_a_known_gap_naming_an_unknown_cell_stops_the_run( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +) -> None: + typo: Final = GAP_MANIFEST.replace("provider: openai, reason: the proxy has no search route", "provider: opneai, reason: r") + + run, results = _run_cells(pytester, monkeypatch, typo) + + assert run.ret != pytest.ExitCode.OK + assert "unknown provider 'opneai'" in run.stdout.str() + run.stderr.str() + assert not results.exists() diff --git a/tests/e2e_harness/client_apps/test_compat_matrix_builder.py b/tests/e2e_harness/client_apps/test_compat_matrix_builder.py index 13b199ead8b..0fb4d851250 100644 --- a/tests/e2e_harness/client_apps/test_compat_matrix_builder.py +++ b/tests/e2e_harness/client_apps/test_compat_matrix_builder.py @@ -6,11 +6,13 @@ from pathlib import Path from typing import Final import pytest +from pydantic import ValidationError from client_apps.compat_results import Failed, NotApplicable, NotTested, Outcome, Passed, ResultRow, write_rows from client_apps.matrix_builder import ( REGRESSION_EXIT_CODE, REGRESSIONS, + KnownGap, Manifest, ManifestFeature, Matrix, @@ -136,3 +138,59 @@ def test_shipped_manifests_validate_and_pin_their_client() -> None: manifest = load_manifest(path) assert len({feature.id for feature in manifest.features}) == len(manifest.features) assert pinned_client_version(manifest.package) + + + +def _manifest_yaml(known_gaps: str) -> str: + return f"""schema_version: "1" +client: codex +package: "@openai/codex" +providers: [openai, vertex_ai] +features: + - {{id: vision, name: Vision}} + - {{id: web_search, name: Web search}} +known_gaps: {known_gaps} +""" + + +def test_known_gaps_are_read_from_the_manifest_yaml(tmp_path: Path) -> None: + path: Final = tmp_path / "manifest.yaml" + _ = path.write_text(_manifest_yaml("[{feature: web_search, provider: vertex_ai, reason: drops search tools}]")) + + manifest: Final = load_manifest(path) + + assert manifest.known_gap("web_search", "vertex_ai") == KnownGap( + feature="web_search", provider="vertex_ai", reason="drops search tools" + ) + assert manifest.known_gap("web_search", "openai") is None + assert manifest.known_gap("vision", "vertex_ai") is None + assert MANIFEST.known_gaps == () + + +@pytest.mark.parametrize( + ("known_gaps", "problem"), + ( + ("[{feature: web_serch, provider: openai, reason: r}]", "unknown feature 'web_serch'"), + ("[{feature: vision, provider: azure, reason: r}]", "unknown provider 'azure'"), + ( + "[{feature: vision, provider: openai, reason: a}, {feature: vision, provider: openai, reason: b}]", + "known_gaps lists a cell more than once: vision/openai", + ), + ), +) +def test_a_known_gap_must_name_a_single_manifest_cell(tmp_path: Path, known_gaps: str, problem: str) -> None: + path: Final = tmp_path / "manifest.yaml" + _ = path.write_text(_manifest_yaml(known_gaps)) + + with pytest.raises(ValidationError, match=problem): + _ = load_manifest(path) + + +def test_a_known_gap_needs_a_reason(tmp_path: Path) -> None: + path: Final = tmp_path / "manifest.yaml" + _ = path.write_text(_manifest_yaml("[{feature: vision, provider: openai, reason: ''}]")) + + with pytest.raises(ValidationError) as raised: + _ = load_manifest(path) + + assert [error["loc"] for error in raised.value.errors()] == [("known_gaps", 0, "reason")]