diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index 7843c1a4091..b654d2dc4ef 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -50,7 +50,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke. `pr_gate_version_resolver.py` only forwards to `cli_harness/npm_version_resolver.py`, because the Buildkite runner images still run that path and install @latest without it - `cli_harness/` - no tests, only the plumbing every suite that drives a real coding-agent CLI shares: `npm_version_resolver.py` picks the CLI release to install (the newest npm release at least three days old, run by CI as `python tests/e2e/cli_harness/npm_version_resolver.py `, stdlib only), `lockdown.py` gives the CLI subprocess an allowlisted env and a fresh empty HOME so it never sees the runner's credentials or dotfiles, and `rate_limiter.py` is the cross-process token bucket per upstream that keeps parallel CLI runs under one request rate. Covered by `tests/e2e_harness/cli_harness/`. Code only one CLI suite needs stays in that suite -- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, and auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/` +- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted, and a model switch, where Codex reasons on the setup's model, then resumes the thread on a second model and is asked for the value after the file is deleted), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(, reason=...)`, which is `SETUPS` with a strict xfail on that one setup, and a row that drives a second model wraps its setups in `also_driving`, which adds that model and its provider to each setup's `Subject`. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` (through `register_model`, which a row uses for any extra model) and the `codex` fixture makes a key scoped to `key_models`, the setup's model unless a row overrides it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts or about the model a turn ran on, so `rollout.py` reads the compactions and each turn's model from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens, and `reasoning_effort`, which sets Codex's `model_reasoning_effort` so the model reasons and the history carries its reasoning. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/` - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream, see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport. The Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`. Its specs follow the dashboard's rule against testing what shadcn and Base UI already guarantee, in `ui/litellm-dashboard/AGENTS.md` ## MCP suite: real Datadog only @@ -315,6 +315,8 @@ other... ``` ## Hard Rules +- no comments in code unless they are truly necessary + - no unit tests of a product feature under `tests/e2e`, and no mock tests or monkeypatching of code anywhere in it: a product feature is proven end to end against a live proxy, never with a unit test. if a contributor asks you to write an end to end test, do NOT stage a unit test with it. If you find a product gap, call it out in the PR description. the harness's own plumbing is tested outside the suite, in `tests/e2e_harness/` (mirroring this folder's layout), because the Buildkite e2e run copies `tests/e2e/` into the runner image and runs every test in it, so a harness test in here would count as a product test in the nightly numbers. those tests run without a proxy and take their inputs as arguments or env vars (setting an env var through pytest's `monkeypatch` fixture is fine, patching a function, class, or module is not), and no coverage-registry or compat-matrix cell rests on them. judge a change inside one of them by that standard, not as a misplaced product test - use model management endpoints to create new models for a test. this could be in a conftest / inline for each test. ask the user what they want. diff --git a/tests/e2e/codex/conftest.py b/tests/e2e/codex/conftest.py index 1ce6f2009a5..1982f15adb4 100644 --- a/tests/e2e/codex/conftest.py +++ b/tests/e2e/codex/conftest.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Iterator +from collections.abc import Callable, Iterator from dataclasses import dataclass from typing import Final @@ -10,6 +10,7 @@ from codex.harness import CodexHarness, codex_harness from codex.setups import Deployment, ModelSetup from e2e_config import PROXY_BASE_URL, unique_marker from lifecycle import ResourceManager +from models import LiteLLMParamsBody from proxy_client import ProxyClient @@ -24,18 +25,31 @@ def client(proxy: ProxyClient) -> _Client: @pytest.fixture -def setup(deployment: Deployment, proxy: ProxyClient, resources: ResourceManager) -> ModelSetup: - model: Final = f"e2e-codex-{unique_marker()}" - model_id: Final = proxy.create_model(model, deployment.litellm_params, provider_live=True) - resources.defer(lambda: proxy.delete_model(model_id)) - return ModelSetup(model=model, rate_bucket=deployment.rate_bucket) +def register_model(proxy: ProxyClient, resources: ResourceManager) -> Callable[[LiteLLMParamsBody], str]: + def register(litellm_params: LiteLLMParamsBody) -> str: + model: Final = f"e2e-codex-{unique_marker()}" + model_id: Final = proxy.create_model(model, litellm_params, provider_live=True) + resources.defer(lambda: proxy.delete_model(model_id)) + return model + + return register @pytest.fixture -def codex(setup: ModelSetup, resources: ResourceManager) -> Iterator[CodexHarness]: +def setup(deployment: Deployment, register_model: Callable[[LiteLLMParamsBody], str]) -> ModelSetup: + return ModelSetup(model=register_model(deployment.litellm_params), rate_bucket=deployment.rate_bucket) + + +@pytest.fixture +def key_models(setup: ModelSetup) -> tuple[str, ...]: + return (setup.model,) + + +@pytest.fixture +def codex(setup: ModelSetup, key_models: tuple[str, ...], resources: ResourceManager) -> Iterator[CodexHarness]: with codex_harness( gateway=PROXY_BASE_URL, - api_key=resources.key(models=[setup.model], user_id=None), + api_key=resources.key(models=list(key_models), user_id=None), rate_bucket=setup.rate_bucket, limiter=get_default_limiter(), ) as harness: diff --git a/tests/e2e/codex/harness.py b/tests/e2e/codex/harness.py index f6dd3700149..1babe6fba52 100644 --- a/tests/e2e/codex/harness.py +++ b/tests/e2e/codex/harness.py @@ -7,11 +7,11 @@ from contextlib import contextmanager from dataclasses import dataclass, field from itertools import chain from pathlib import Path -from typing import Final +from typing import Final, Literal from cli_harness.lockdown import isolated_home, locked_down_env from cli_harness.rate_limiter import RateBucket, RateLimiter -from codex.rollout import Compaction, rollout_compactions +from codex.rollout import Compaction, rollout_compactions, rollout_turn_models from codex.run import CodexRun, codex_run from e2e_metadata import step @@ -19,6 +19,7 @@ CODEX_CLI: Final = "codex" EXEC_TIMEOUT_SECONDS: Final = 600.0 _API_KEY_ENV: Final = "LITELLM_API_KEY" _PROVIDER: Final = "litellm" +type ReasoningEffort = Literal["low", "medium", "high"] _CONFIG_TOML: Final = """\ model_provider = "{provider}" approval_policy = "never" @@ -58,22 +59,43 @@ class CodexHarness: return workspace @step('Run Codex on {model} through the proxy with the prompt "{prompt}"') - def exec(self, prompt: str, *, model: str, cwd: Path, compact_after_tokens: int | None = None) -> CodexRun: - return self._run(("exec",), (*_compaction_flags(compact_after_tokens), "--model", model, "--", prompt), cwd) + def exec( + self, + prompt: str, + *, + model: str, + cwd: Path, + compact_after_tokens: int | None = None, + reasoning_effort: ReasoningEffort | None = None, + ) -> CodexRun: + return self._run(("exec",), (*_turn_flags(model, compact_after_tokens, reasoning_effort), "--", prompt), cwd) @step('Resume Codex thread {thread_id} on {model} through the proxy with the prompt "{prompt}"') def resume( - self, thread_id: str, prompt: str, *, model: str, cwd: Path, compact_after_tokens: int | None = None + self, + thread_id: str, + prompt: str, + *, + model: str, + cwd: Path, + compact_after_tokens: int | None = None, + reasoning_effort: ReasoningEffort | None = None, ) -> CodexRun: return self._run( ("exec", "resume"), - (*_compaction_flags(compact_after_tokens), "--model", model, "--", thread_id, prompt), + (*_turn_flags(model, compact_after_tokens, reasoning_effort), "--", thread_id, prompt), cwd, ) def compactions(self, thread_id: str) -> tuple[Compaction, ...]: - rollouts: Final = sorted(self.codex_home.glob(f"sessions/**/rollout-*-{thread_id}.jsonl")) - return tuple(chain.from_iterable(rollout_compactions(rollout.read_text()) for rollout in rollouts)) + return tuple(chain.from_iterable(rollout_compactions(rollout) for rollout in self._rollouts(thread_id))) + + def turn_models(self, thread_id: str) -> tuple[str, ...]: + return tuple(chain.from_iterable(rollout_turn_models(rollout) for rollout in self._rollouts(thread_id))) + + def _rollouts(self, thread_id: str) -> tuple[str, ...]: + paths: Final = sorted(self.codex_home.glob(f"sessions/**/rollout-*-{thread_id}.jsonl")) + return tuple(path.read_text() for path in paths) def _run(self, subcommand: tuple[str, ...], args: tuple[str, ...], cwd: Path) -> CodexRun: _ = self.limiter.acquire(self.rate_bucket) @@ -107,6 +129,16 @@ def _compaction_flags(after_tokens: int | None) -> tuple[str, ...]: ) +def _turn_flags( + model: str, compact_after_tokens: int | None, reasoning_effort: ReasoningEffort | None +) -> tuple[str, ...]: + return (*_compaction_flags(compact_after_tokens), *_reasoning_flags(reasoning_effort), "--model", model) + + +def _reasoning_flags(effort: ReasoningEffort | None) -> tuple[str, ...]: + return () if effort is None else ("--config", f'model_reasoning_effort="{effort}"') + + @step("Give Codex a fresh empty home whose config sends every request to the proxy") @contextmanager def codex_harness( diff --git a/tests/e2e/codex/rollout.py b/tests/e2e/codex/rollout.py index fcaeaa7ca42..3a344530bff 100644 --- a/tests/e2e/codex/rollout.py +++ b/tests/e2e/codex/rollout.py @@ -1,9 +1,9 @@ -"""The compactions Codex recorded in a thread's rollout file, `CODEX_HOME/sessions/**/rollout-*-.jsonl`. +"""What Codex recorded in a thread's rollout file, `CODEX_HOME/sessions/**/rollout-*-.jsonl`. -`codex exec --json` prints no event for a compaction, so the rollout is the only place it shows. Each line is -`{"timestamp", "type", "payload"}` (`codex-rs/history/src/rollout_payload.rs`) and only `compacted` lines are read. -A remote compaction is one whose replacement history carries the provider's `compaction` item, while a local one -replaces the history with a summary Codex wrote itself. +`codex exec --json` prints no event for a compaction or for the model a turn ran on, so the rollout is the only place +they show. Each line is `{"timestamp", "type", "payload"}` (`codex-rs/history/src/rollout_payload.rs`) and only +`compacted` and `turn_context` lines are read. A remote compaction is one whose replacement history carries the +provider's `compaction` item, while a local one replaces the history with a summary Codex wrote itself. """ from __future__ import annotations @@ -14,6 +14,7 @@ from typing import Final from pydantic import BaseModel, ConfigDict _COMPACTED: Final = "compacted" +_TURN_CONTEXT: Final = "turn_context" _PROVIDER_COMPACTION_ITEM: Final = "compaction" @@ -34,14 +35,27 @@ class _CompactedPayload(_Frozen): replacement_history: tuple[_HistoryItem, ...] | None = None +class _TurnContextPayload(_Frozen): + model: str + + @dataclass(frozen=True, slots=True) class Compaction: remote: bool def rollout_compactions(rollout: str) -> tuple[Compaction, ...]: - lines: Final = tuple(_Line.model_validate_json(line) for line in rollout.splitlines() if line.strip()) - return tuple(_compaction(line.payload) for line in lines if line.type == _COMPACTED) + return tuple(_compaction(line.payload) for line in _lines(rollout) if line.type == _COMPACTED) + + +def rollout_turn_models(rollout: str) -> tuple[str, ...]: + return tuple( + _TurnContextPayload.model_validate(line.payload).model for line in _lines(rollout) if line.type == _TURN_CONTEXT + ) + + +def _lines(rollout: str) -> tuple[_Line, ...]: + return tuple(_Line.model_validate_json(line) for line in rollout.splitlines() if line.strip()) def _compaction(payload: object) -> Compaction: diff --git a/tests/e2e/codex/setups.py b/tests/e2e/codex/setups.py index 4716706c51c..19c92fae51b 100644 --- a/tests/e2e/codex/setups.py +++ b/tests/e2e/codex/setups.py @@ -3,7 +3,8 @@ from __future__ import annotations -from dataclasses import dataclass +from collections.abc import Mapping +from dataclasses import dataclass, replace from typing import Final, Literal import pytest @@ -17,7 +18,7 @@ GPT_ON_BEDROCK_RUNTIME: Final = "bedrock/us.openai.gpt-6.1-sol" GPT_ON_AZURE_OPENAI: Final = "azure/gpt-6.1-sol" GPT_ON_OPENAI: Final = "openai/gpt-6.1-sol" CLAUDE_ON_BEDROCK: Final = "bedrock/us.anthropic.claude-sonnet-5-5" -_BEDROCK_REGION: Final = "us-east-1" +BEDROCK_REGION: Final = "us-east-1" type SetupId = Literal["bedrock_mantle", "bedrock_runtime", "azure_openai", "openai", "claude_bedrock"] @@ -49,14 +50,14 @@ def _drives(provider: Provider, model: str) -> pytest.MarkDecorator: SETUPS: Final = ( pytest.param( Deployment( - LiteLLMParamsBody(model=GPT_ON_BEDROCK_MANTLE, aws_region_name=_BEDROCK_REGION), RateBucket.BEDROCK_MANTLE + LiteLLMParamsBody(model=GPT_ON_BEDROCK_MANTLE, aws_region_name=BEDROCK_REGION), RateBucket.BEDROCK_MANTLE ), id="bedrock_mantle", marks=_drives(Provider.BEDROCK_MANTLE, GPT_ON_BEDROCK_MANTLE), ), pytest.param( Deployment( - LiteLLMParamsBody(model=GPT_ON_BEDROCK_RUNTIME, aws_region_name=_BEDROCK_REGION), + LiteLLMParamsBody(model=GPT_ON_BEDROCK_RUNTIME, aws_region_name=BEDROCK_REGION), RateBucket.BEDROCK_CONVERSE, ), id="bedrock_runtime", @@ -79,7 +80,7 @@ SETUPS: Final = ( ), pytest.param( Deployment( - LiteLLMParamsBody(model=CLAUDE_ON_BEDROCK, aws_region_name=_BEDROCK_REGION), RateBucket.BEDROCK_CONVERSE + LiteLLMParamsBody(model=CLAUDE_ON_BEDROCK, aws_region_name=BEDROCK_REGION), RateBucket.BEDROCK_CONVERSE ), id="claude_bedrock", marks=_drives(Provider.BEDROCK, CLAUDE_ON_BEDROCK), @@ -94,3 +95,33 @@ def with_expected_failure(setup_id: SetupId, *, reason: str) -> tuple[ParameterS else setup for setup in SETUPS ) + + +@dataclass(frozen=True, slots=True) +class ExtraModel: + provider: Provider + litellm_params: LiteLLMParamsBody + + +def also_driving(setups: tuple[ParameterSet, ...], extra: Mapping[str, ExtraModel]) -> tuple[ParameterSet, ...]: + return tuple(_also_driving(setup, extra) for setup in setups) + + +def _also_driving(setup: ParameterSet, extra: Mapping[str, ExtraModel]) -> ParameterSet: + deployment: Final = setup.values[0] + assert isinstance(deployment, Deployment) + added: Final = extra[deployment.litellm_params.model] + return pytest.param(*setup.values, id=setup.id, marks=tuple(_widened(mark, added) for mark in setup.marks)) + + +def _widened(mark: pytest.MarkDecorator | pytest.Mark, added: ExtraModel) -> pytest.MarkDecorator | pytest.Mark: + subject: Final = mark.args[0] if mark.name == "meta" and mark.args else None + if not isinstance(subject, Subject): + return mark + return meta( + replace( + subject, + providers=(*subject.providers, added.provider), + models=(*subject.models, added.litellm_params.model), + ) + ) diff --git a/tests/e2e/codex/test_switch_model_e2e.py b/tests/e2e/codex/test_switch_model_e2e.py new file mode 100644 index 00000000000..ea8225ce281 --- /dev/null +++ b/tests/e2e/codex/test_switch_model_e2e.py @@ -0,0 +1,99 @@ +"""Azure has a single GPT deployment on the test stack, so the Azure setup switches to another GPT on OpenAI.""" + +from __future__ import annotations + +from collections.abc import Callable +from types import MappingProxyType +from typing import Final + +import pytest +from codex.harness import CodexHarness +from codex.setups import ( + BEDROCK_REGION, + CLAUDE_ON_BEDROCK, + GPT_ON_AZURE_OPENAI, + GPT_ON_BEDROCK_MANTLE, + GPT_ON_BEDROCK_RUNTIME, + GPT_ON_OPENAI, + Deployment, + ExtraModel, + ModelSetup, + also_driving, + with_expected_failure, +) +from e2e_config import unique_marker +from e2e_metadata import Provider +from models import LiteLLMParamsBody + +pytestmark = pytest.mark.e2e + +TERRA_ON_OPENAI: Final = ExtraModel( + Provider.OPENAI, LiteLLMParamsBody(model="openai/gpt-6.1-terra", api_key="os.environ/OPENAI_API_KEY") +) +SWITCH_TO: Final = MappingProxyType( + { + GPT_ON_BEDROCK_MANTLE: ExtraModel( + Provider.BEDROCK_MANTLE, + LiteLLMParamsBody(model="bedrock_mantle/openai.gpt-6-luna", aws_region_name=BEDROCK_REGION), + ), + GPT_ON_BEDROCK_RUNTIME: ExtraModel( + Provider.BEDROCK, LiteLLMParamsBody(model="bedrock/us.openai.gpt-6-luna", aws_region_name=BEDROCK_REGION) + ), + GPT_ON_AZURE_OPENAI: TERRA_ON_OPENAI, + GPT_ON_OPENAI: TERRA_ON_OPENAI, + CLAUDE_ON_BEDROCK: ExtraModel( + Provider.BEDROCK, + LiteLLMParamsBody(model="bedrock/us.anthropic.claude-haiku-5-5", aws_region_name=BEDROCK_REGION), + ), + } +) + + +@pytest.fixture +def second_model(deployment: Deployment, register_model: Callable[[LiteLLMParamsBody], str]) -> str: + return register_model(SWITCH_TO[deployment.litellm_params.model].litellm_params) + + +@pytest.fixture +def key_models(setup: ModelSetup, second_model: str) -> tuple[str, ...]: + return (setup.model, second_model) + + +@pytest.mark.parametrize( + "deployment", + also_driving( + with_expected_failure( + "azure_openai", + reason="LiteLLM forwards the reasoning Azure encrypted to OpenAI, which cannot decrypt another account's" + " reasoning and rejects the turn with invalid_encrypted_content (#41792)", + ), + SWITCH_TO, + ), +) +class TestCodexSwitchModel: + def test_remembers_a_value_after_resuming_on_another_model( + self, codex: CodexHarness, setup: ModelSetup, second_model: str + ) -> None: + marker: Final = unique_marker() + workspace: Final = codex.new_workspace(files={"secret.txt": marker}) + first: Final = codex.exec( + "Run `cat secret.txt`. Remember the value it prints, then work out how many of its characters are digits" + " and reply with that count.", + model=setup.model, + cwd=workspace, + reasoning_effort="high", + ) + assert first.thread_id is not None, first + assert any(marker in command.output for command in first.commands), first + assert first.usage is not None and first.usage.reasoning_output_tokens > 0, first + (workspace / "secret.txt").unlink() + second: Final = codex.resume( + first.thread_id, + "What value did secret.txt print earlier? Reply with just that value.", + model=second_model, + cwd=workspace, + reasoning_effort="high", + ) + assert second.exit_code == 0, second + assert codex.turn_models(first.thread_id) == (setup.model, second_model), second + assert marker in second.answer, second diff --git a/tests/e2e_harness/AGENTS.md b/tests/e2e_harness/AGENTS.md index 83e573ae249..160a35c8c3b 100644 --- a/tests/e2e_harness/AGENTS.md +++ b/tests/e2e_harness/AGENTS.md @@ -2,7 +2,7 @@ Tests of the harness under `tests/e2e/` (the transport, the clients, the fixture bundle and replay edge, the stack lock, the IdP launcher, the coverage collector, the JUnit properties and the Claude Code driver), not of the product. They live outside `tests/e2e/` because the Buildkite e2e run copies that folder into the runner image and runs every test in it, so a harness test in there counts as a product test in the nightly numbers. Nothing here needs a proxy, provider keys or the network -The layout mirrors `tests/e2e/`: `test_e2e_http.py` covers `tests/e2e/e2e_http.py`, `logging/test_datadog_reader.py` covers `tests/e2e/logging/datadog_reader.py`, `claude_code/` covers the driver, builder and probe, `cli_harness/` covers the version resolver, the CLI env lockdown and the rate limiter, and `codex/` covers how the Codex driver reads the CLI's JSONL events and the compactions in a thread's rollout file. Put a new harness test under the folder that mirrors the suite folder whose module it covers +The layout mirrors `tests/e2e/`: `test_e2e_http.py` covers `tests/e2e/e2e_http.py`, `logging/test_datadog_reader.py` covers `tests/e2e/logging/datadog_reader.py`, `claude_code/` covers the driver, builder and probe, `cli_harness/` covers the version resolver, the CLI env lockdown and the rate limiter, and `codex/` covers how the Codex driver reads the CLI's JSONL events and a thread's rollout file, and how a row adds a second model to its setups' metadata. Put a new harness test under the folder that mirrors the suite folder whose module it covers Run them from the repo root. `e2e_config` reads `LITELLM_MASTER_KEY` at import and any value will do, the CI lane sets a dummy: diff --git a/tests/e2e_harness/codex/test_codex_rollout.py b/tests/e2e_harness/codex/test_codex_rollout.py index 2ef27a8a9c5..902dd77a8d5 100644 --- a/tests/e2e_harness/codex/test_codex_rollout.py +++ b/tests/e2e_harness/codex/test_codex_rollout.py @@ -8,7 +8,7 @@ from typing import Final from cli_harness.rate_limiter import RateBucket, RateLimiter from codex.harness import CodexHarness -from codex.rollout import Compaction, rollout_compactions +from codex.rollout import Compaction, rollout_compactions, rollout_turn_models THREAD_ID: Final = "01a18fea-b2e8-7ed2-a29b-1ce4d2b138c3" USER_PROMPT: Final = { @@ -60,6 +60,19 @@ class TestRolloutCompactions: assert rollout_compactions(_rollout()) == () +def test_turn_models_are_the_models_each_turn_ran_on_in_order() -> None: + rollout: Final = "\n".join( + ( + _line("turn_context", {"turn_id": "turn-1", "model": "first-model", "effort": "high"}), + _line("response_item", USER_PROMPT), + _line("compacted", REMOTE), + _line("turn_context", {"turn_id": "turn-2", "model": "second-model", "effort": "high"}), + ) + ) + + assert rollout_turn_models(rollout) == ("first-model", "second-model") + + def test_the_harness_reads_only_the_rollout_of_the_thread_it_is_asked_about(tmp_path: Path) -> None: codex: Final = CodexHarness( home=tmp_path, diff --git a/tests/e2e_harness/codex/test_codex_setups.py b/tests/e2e_harness/codex/test_codex_setups.py new file mode 100644 index 00000000000..40f4522635d --- /dev/null +++ b/tests/e2e_harness/codex/test_codex_setups.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from types import MappingProxyType +from typing import Final + +from _pytest.mark.structures import ParameterSet +from codex.setups import SETUPS, Deployment, ExtraModel, also_driving, with_expected_failure +from e2e_metadata import Provider, Subject +from models import LiteLLMParamsBody + +SECOND_MODEL: Final = "openai/second-model" + + +def _subject(setup: ParameterSet) -> Subject: + (meta,) = (mark for mark in setup.marks if mark.name == "meta") + return _only_subject(meta.args) + + +def _only_subject(args: tuple[object, ...]) -> Subject: + (subject,) = args + assert isinstance(subject, Subject) + return subject + + +def _model(setup: ParameterSet) -> str: + deployment: Final = setup.values[0] + assert isinstance(deployment, Deployment) + return deployment.litellm_params.model + + +def test_also_driving_declares_the_extra_model_on_every_setup_and_keeps_the_xfail() -> None: + second: Final = ExtraModel(Provider.OPENAI, LiteLLMParamsBody(model=SECOND_MODEL)) + setups: Final = also_driving( + with_expected_failure("azure_openai", reason="known bug"), + MappingProxyType({_model(setup): second for setup in SETUPS}), + ) + + assert tuple(setup.id for setup in setups) == tuple(setup.id for setup in SETUPS) + assert all(_subject(setup).models == tuple(sorted((_model(setup), SECOND_MODEL))) for setup in setups) + azure: Final = next(setup for setup in setups if setup.id == "azure_openai") + assert _subject(azure).providers == (Provider.AZURE, Provider.OPENAI) + assert any(mark.name == "xfail" for mark in azure.marks)