From 71cf8794e67192ef8b1e0384afd70145c83f3d0d Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 2 Nov 2026 23:06:39 -0800 Subject: [PATCH] test(e2e): assert codex really resumed on the second model when switching models --- tests/e2e/AGENTS.md | 2 +- tests/e2e/codex/harness.py | 41 ++++++++----------- tests/e2e/codex/rollout.py | 28 +++++++++---- tests/e2e/codex/test_switch_model_e2e.py | 1 + tests/e2e_harness/codex/test_codex_rollout.py | 15 ++++++- 5 files changed, 54 insertions(+), 33 deletions(-) diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index d6716d019e4..96bb617ee66 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -50,7 +50,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke. `pr_gate_version_resolver.py` only forwards to `cli_harness/npm_version_resolver.py`, because the Buildkite runner images still run that path and install @latest without it - `cli_harness/` - no tests, only the plumbing every suite that drives a real coding-agent CLI shares: `npm_version_resolver.py` picks the CLI release to install (the newest npm release at least three days old, run by CI as `python tests/e2e/cli_harness/npm_version_resolver.py `, stdlib only), `lockdown.py` gives the CLI subprocess an allowlisted env and a fresh empty HOME so it never sees the runner's credentials or dotfiles, and `rate_limiter.py` is the cross-process token bucket per upstream that keeps parallel CLI runs under one request rate. Covered by `tests/e2e_harness/cli_harness/`. Code only one CLI suite needs stays in that suite -- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted, and a model switch, where Codex reasons on the setup's model, then resumes the thread on a second model and is asked for the value after the file is deleted), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` (through `register_model`, which a row uses for any extra model) and the `codex` fixture makes a key scoped to `key_models`, the setup's model unless a row overrides it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens, and `reasoning_effort`, which sets Codex's `model_reasoning_effort` so the model reasons and the history carries its reasoning. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/` +- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted, and a model switch, where Codex reasons on the setup's model, then resumes the thread on a second model and is asked for the value after the file is deleted), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` (through `register_model`, which a row uses for any extra model) and the `codex` fixture makes a key scoped to `key_models`, the setup's model unless a row overrides it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts or about the model a turn ran on, so `rollout.py` reads the compactions and each turn's model from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens, and `reasoning_effort`, which sets Codex's `model_reasoning_effort` so the model reasons and the history carries its reasoning. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/` - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream, see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport. The Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`. Its specs follow the dashboard's rule against testing what shadcn and Base UI already guarantee, in `ui/litellm-dashboard/AGENTS.md` ## MCP suite: real Datadog only diff --git a/tests/e2e/codex/harness.py b/tests/e2e/codex/harness.py index f7907d1d661..1babe6fba52 100644 --- a/tests/e2e/codex/harness.py +++ b/tests/e2e/codex/harness.py @@ -11,7 +11,7 @@ from typing import Final, Literal from cli_harness.lockdown import isolated_home, locked_down_env from cli_harness.rate_limiter import RateBucket, RateLimiter -from codex.rollout import Compaction, rollout_compactions +from codex.rollout import Compaction, rollout_compactions, rollout_turn_models from codex.run import CodexRun, codex_run from e2e_metadata import step @@ -68,18 +68,7 @@ class CodexHarness: compact_after_tokens: int | None = None, reasoning_effort: ReasoningEffort | None = None, ) -> CodexRun: - return self._run( - ("exec",), - ( - *_compaction_flags(compact_after_tokens), - *_reasoning_flags(reasoning_effort), - "--model", - model, - "--", - prompt, - ), - cwd, - ) + return self._run(("exec",), (*_turn_flags(model, compact_after_tokens, reasoning_effort), "--", prompt), cwd) @step('Resume Codex thread {thread_id} on {model} through the proxy with the prompt "{prompt}"') def resume( @@ -94,21 +83,19 @@ class CodexHarness: ) -> CodexRun: return self._run( ("exec", "resume"), - ( - *_compaction_flags(compact_after_tokens), - *_reasoning_flags(reasoning_effort), - "--model", - model, - "--", - thread_id, - prompt, - ), + (*_turn_flags(model, compact_after_tokens, reasoning_effort), "--", thread_id, prompt), cwd, ) def compactions(self, thread_id: str) -> tuple[Compaction, ...]: - rollouts: Final = sorted(self.codex_home.glob(f"sessions/**/rollout-*-{thread_id}.jsonl")) - return tuple(chain.from_iterable(rollout_compactions(rollout.read_text()) for rollout in rollouts)) + return tuple(chain.from_iterable(rollout_compactions(rollout) for rollout in self._rollouts(thread_id))) + + def turn_models(self, thread_id: str) -> tuple[str, ...]: + return tuple(chain.from_iterable(rollout_turn_models(rollout) for rollout in self._rollouts(thread_id))) + + def _rollouts(self, thread_id: str) -> tuple[str, ...]: + paths: Final = sorted(self.codex_home.glob(f"sessions/**/rollout-*-{thread_id}.jsonl")) + return tuple(path.read_text() for path in paths) def _run(self, subcommand: tuple[str, ...], args: tuple[str, ...], cwd: Path) -> CodexRun: _ = self.limiter.acquire(self.rate_bucket) @@ -142,6 +129,12 @@ def _compaction_flags(after_tokens: int | None) -> tuple[str, ...]: ) +def _turn_flags( + model: str, compact_after_tokens: int | None, reasoning_effort: ReasoningEffort | None +) -> tuple[str, ...]: + return (*_compaction_flags(compact_after_tokens), *_reasoning_flags(reasoning_effort), "--model", model) + + def _reasoning_flags(effort: ReasoningEffort | None) -> tuple[str, ...]: return () if effort is None else ("--config", f'model_reasoning_effort="{effort}"') diff --git a/tests/e2e/codex/rollout.py b/tests/e2e/codex/rollout.py index fcaeaa7ca42..3a344530bff 100644 --- a/tests/e2e/codex/rollout.py +++ b/tests/e2e/codex/rollout.py @@ -1,9 +1,9 @@ -"""The compactions Codex recorded in a thread's rollout file, `CODEX_HOME/sessions/**/rollout-*-.jsonl`. +"""What Codex recorded in a thread's rollout file, `CODEX_HOME/sessions/**/rollout-*-.jsonl`. -`codex exec --json` prints no event for a compaction, so the rollout is the only place it shows. Each line is -`{"timestamp", "type", "payload"}` (`codex-rs/history/src/rollout_payload.rs`) and only `compacted` lines are read. -A remote compaction is one whose replacement history carries the provider's `compaction` item, while a local one -replaces the history with a summary Codex wrote itself. +`codex exec --json` prints no event for a compaction or for the model a turn ran on, so the rollout is the only place +they show. Each line is `{"timestamp", "type", "payload"}` (`codex-rs/history/src/rollout_payload.rs`) and only +`compacted` and `turn_context` lines are read. A remote compaction is one whose replacement history carries the +provider's `compaction` item, while a local one replaces the history with a summary Codex wrote itself. """ from __future__ import annotations @@ -14,6 +14,7 @@ from typing import Final from pydantic import BaseModel, ConfigDict _COMPACTED: Final = "compacted" +_TURN_CONTEXT: Final = "turn_context" _PROVIDER_COMPACTION_ITEM: Final = "compaction" @@ -34,14 +35,27 @@ class _CompactedPayload(_Frozen): replacement_history: tuple[_HistoryItem, ...] | None = None +class _TurnContextPayload(_Frozen): + model: str + + @dataclass(frozen=True, slots=True) class Compaction: remote: bool def rollout_compactions(rollout: str) -> tuple[Compaction, ...]: - lines: Final = tuple(_Line.model_validate_json(line) for line in rollout.splitlines() if line.strip()) - return tuple(_compaction(line.payload) for line in lines if line.type == _COMPACTED) + return tuple(_compaction(line.payload) for line in _lines(rollout) if line.type == _COMPACTED) + + +def rollout_turn_models(rollout: str) -> tuple[str, ...]: + return tuple( + _TurnContextPayload.model_validate(line.payload).model for line in _lines(rollout) if line.type == _TURN_CONTEXT + ) + + +def _lines(rollout: str) -> tuple[_Line, ...]: + return tuple(_Line.model_validate_json(line) for line in rollout.splitlines() if line.strip()) def _compaction(payload: object) -> Compaction: diff --git a/tests/e2e/codex/test_switch_model_e2e.py b/tests/e2e/codex/test_switch_model_e2e.py index 87c572dcd58..e6bba0f864d 100644 --- a/tests/e2e/codex/test_switch_model_e2e.py +++ b/tests/e2e/codex/test_switch_model_e2e.py @@ -83,4 +83,5 @@ class TestCodexSwitchModel: reasoning_effort="high", ) assert second.exit_code == 0, second + assert codex.turn_models(first.thread_id) == (setup.model, second_model), second assert marker in second.answer, second diff --git a/tests/e2e_harness/codex/test_codex_rollout.py b/tests/e2e_harness/codex/test_codex_rollout.py index 2ef27a8a9c5..902dd77a8d5 100644 --- a/tests/e2e_harness/codex/test_codex_rollout.py +++ b/tests/e2e_harness/codex/test_codex_rollout.py @@ -8,7 +8,7 @@ from typing import Final from cli_harness.rate_limiter import RateBucket, RateLimiter from codex.harness import CodexHarness -from codex.rollout import Compaction, rollout_compactions +from codex.rollout import Compaction, rollout_compactions, rollout_turn_models THREAD_ID: Final = "01a18fea-b2e8-7ed2-a29b-1ce4d2b138c3" USER_PROMPT: Final = { @@ -60,6 +60,19 @@ class TestRolloutCompactions: assert rollout_compactions(_rollout()) == () +def test_turn_models_are_the_models_each_turn_ran_on_in_order() -> None: + rollout: Final = "\n".join( + ( + _line("turn_context", {"turn_id": "turn-1", "model": "first-model", "effort": "high"}), + _line("response_item", USER_PROMPT), + _line("compacted", REMOTE), + _line("turn_context", {"turn_id": "turn-2", "model": "second-model", "effort": "high"}), + ) + ) + + assert rollout_turn_models(rollout) == ("first-model", "second-model") + + def test_the_harness_reads_only_the_rollout_of_the_thread_it_is_asked_about(tmp_path: Path) -> None: codex: Final = CodexHarness( home=tmp_path,