test(e2e): assert codex really resumed on the second model when switching models

tests/e2e/AGENTS.md

@@ -50,7 +50,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
 - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests
 - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke. `pr_gate_version_resolver.py` only forwards to `cli_harness/npm_version_resolver.py`, because the Buildkite runner images still run that path and install @latest without it
 - `cli_harness/` - no tests, only the plumbing every suite that drives a real coding-agent CLI shares: `npm_version_resolver.py` picks the CLI release to install (the newest npm release at least three days old, run by CI as `python tests/e2e/cli_harness/npm_version_resolver.py <npm package>`, stdlib only), `lockdown.py` gives the CLI subprocess an allowlisted env and a fresh empty HOME so it never sees the runner's credentials or dotfiles, and `rate_limiter.py` is the cross-process token bucket per upstream that keeps parallel CLI runs under one request rate. Covered by `tests/e2e_harness/cli_harness/`. Code only one CLI suite needs stays in that suite
-- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted, and a model switch, where Codex reasons on the setup's model, then resumes the thread on a second model and is asked for the value after the file is deleted), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(<setup id>, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` (through `register_model`, which a row uses for any extra model) and the `codex` fixture makes a key scoped to `key_models`, the setup's model unless a row overrides it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens, and `reasoning_effort`, which sets Codex's `model_reasoning_effort` so the model reasons and the history carries its reasoning. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/`
+- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted, and a model switch, where Codex reasons on the setup's model, then resumes the thread on a second model and is asked for the value after the file is deleted), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(<setup id>, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` (through `register_model`, which a row uses for any extra model) and the `codex` fixture makes a key scoped to `key_models`, the setup's model unless a row overrides it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts or about the model a turn ran on, so `rollout.py` reads the compactions and each turn's model from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens, and `reasoning_effort`, which sets Codex's `model_reasoning_effort` so the model reasons and the history carries its reasoning. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/`
 - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream, see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport. The Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`. Its specs follow the dashboard's rule against testing what shadcn and Base UI already guarantee, in `ui/litellm-dashboard/AGENTS.md`
 
 ## MCP suite: real Datadog only

tests/e2e/codex/harness.py

@@ -11,7 +11,7 @@ from typing import Final, Literal
 
 from cli_harness.lockdown import isolated_home, locked_down_env
 from cli_harness.rate_limiter import RateBucket, RateLimiter
-from codex.rollout import Compaction, rollout_compactions
+from codex.rollout import Compaction, rollout_compactions, rollout_turn_models
 from codex.run import CodexRun, codex_run
 from e2e_metadata import step
 
@@ -68,18 +68,7 @@ class CodexHarness:
         compact_after_tokens: int | None = None,
         reasoning_effort: ReasoningEffort | None = None,
     ) -> CodexRun:
-        return self._run(
-            ("exec",),
-            (
-                *_compaction_flags(compact_after_tokens),
-                *_reasoning_flags(reasoning_effort),
-                "--model",
-                model,
-                "--",
-                prompt,
-            ),
-            cwd,
-        )
+        return self._run(("exec",), (*_turn_flags(model, compact_after_tokens, reasoning_effort), "--", prompt), cwd)
 
     @step('Resume Codex thread {thread_id} on {model} through the proxy with the prompt "{prompt}"')
     def resume(
@@ -94,21 +83,19 @@ class CodexHarness:
     ) -> CodexRun:
         return self._run(
             ("exec", "resume"),
-            (
-                *_compaction_flags(compact_after_tokens),
-                *_reasoning_flags(reasoning_effort),
-                "--model",
-                model,
-                "--",
-                thread_id,
-                prompt,
-            ),
+            (*_turn_flags(model, compact_after_tokens, reasoning_effort), "--", thread_id, prompt),
             cwd,
         )
 
     def compactions(self, thread_id: str) -> tuple[Compaction, ...]:
-        rollouts: Final = sorted(self.codex_home.glob(f"sessions/**/rollout-*-{thread_id}.jsonl"))
-        return tuple(chain.from_iterable(rollout_compactions(rollout.read_text()) for rollout in rollouts))
+        return tuple(chain.from_iterable(rollout_compactions(rollout) for rollout in self._rollouts(thread_id)))
+
+    def turn_models(self, thread_id: str) -> tuple[str, ...]:
+        return tuple(chain.from_iterable(rollout_turn_models(rollout) for rollout in self._rollouts(thread_id)))
+
+    def _rollouts(self, thread_id: str) -> tuple[str, ...]:
+        paths: Final = sorted(self.codex_home.glob(f"sessions/**/rollout-*-{thread_id}.jsonl"))
+        return tuple(path.read_text() for path in paths)
 
     def _run(self, subcommand: tuple[str, ...], args: tuple[str, ...], cwd: Path) -> CodexRun:
         _ = self.limiter.acquire(self.rate_bucket)
@@ -142,6 +129,12 @@ def _compaction_flags(after_tokens: int | None) -> tuple[str, ...]:
             )
 
 
+def _turn_flags(
+    model: str, compact_after_tokens: int | None, reasoning_effort: ReasoningEffort | None
+) -> tuple[str, ...]:
+    return (*_compaction_flags(compact_after_tokens), *_reasoning_flags(reasoning_effort), "--model", model)
+
+
 def _reasoning_flags(effort: ReasoningEffort | None) -> tuple[str, ...]:
     return () if effort is None else ("--config", f'model_reasoning_effort="{effort}"')
 

tests/e2e/codex/rollout.py

@@ -1,9 +1,9 @@
-"""The compactions Codex recorded in a thread's rollout file, `CODEX_HOME/sessions/**/rollout-*-<thread id>.jsonl`.
+"""What Codex recorded in a thread's rollout file, `CODEX_HOME/sessions/**/rollout-*-<thread id>.jsonl`.
 
-`codex exec --json` prints no event for a compaction, so the rollout is the only place it shows. Each line is
-`{"timestamp", "type", "payload"}` (`codex-rs/history/src/rollout_payload.rs`) and only `compacted` lines are read.
-A remote compaction is one whose replacement history carries the provider's `compaction` item, while a local one
-replaces the history with a summary Codex wrote itself.
+`codex exec --json` prints no event for a compaction or for the model a turn ran on, so the rollout is the only place
+they show. Each line is `{"timestamp", "type", "payload"}` (`codex-rs/history/src/rollout_payload.rs`) and only
+`compacted` and `turn_context` lines are read. A remote compaction is one whose replacement history carries the
+provider's `compaction` item, while a local one replaces the history with a summary Codex wrote itself.
 """
 
 from __future__ import annotations
@@ -14,6 +14,7 @@ from typing import Final
 from pydantic import BaseModel, ConfigDict
 
 _COMPACTED: Final = "compacted"
+_TURN_CONTEXT: Final = "turn_context"
 _PROVIDER_COMPACTION_ITEM: Final = "compaction"
 
 
@@ -34,14 +35,27 @@ class _CompactedPayload(_Frozen):
     replacement_history: tuple[_HistoryItem, ...] | None = None
 
 
+class _TurnContextPayload(_Frozen):
+    model: str
+
+
 @dataclass(frozen=True, slots=True)
 class Compaction:
     remote: bool
 
 
 def rollout_compactions(rollout: str) -> tuple[Compaction, ...]:
-    lines: Final = tuple(_Line.model_validate_json(line) for line in rollout.splitlines() if line.strip())
-    return tuple(_compaction(line.payload) for line in lines if line.type == _COMPACTED)
+    return tuple(_compaction(line.payload) for line in _lines(rollout) if line.type == _COMPACTED)
+
+
+def rollout_turn_models(rollout: str) -> tuple[str, ...]:
+    return tuple(
+        _TurnContextPayload.model_validate(line.payload).model for line in _lines(rollout) if line.type == _TURN_CONTEXT
+    )
+
+
+def _lines(rollout: str) -> tuple[_Line, ...]:
+    return tuple(_Line.model_validate_json(line) for line in rollout.splitlines() if line.strip())
 
 
 def _compaction(payload: object) -> Compaction:

tests/e2e/codex/test_switch_model_e2e.py

@@ -83,4 +83,5 @@ class TestCodexSwitchModel:
             reasoning_effort="high",
         )
         assert second.exit_code == 0, second
+        assert codex.turn_models(first.thread_id) == (setup.model, second_model), second
         assert marker in second.answer, second

tests/e2e_harness/codex/test_codex_rollout.py

@@ -8,7 +8,7 @@ from typing import Final
 
 from cli_harness.rate_limiter import RateBucket, RateLimiter
 from codex.harness import CodexHarness
-from codex.rollout import Compaction, rollout_compactions
+from codex.rollout import Compaction, rollout_compactions, rollout_turn_models
 
 THREAD_ID: Final = "01a18fea-b2e8-7ed2-a29b-1ce4d2b138c3"
 USER_PROMPT: Final = {
@@ -60,6 +60,19 @@ class TestRolloutCompactions:
         assert rollout_compactions(_rollout()) == ()
 
 
+def test_turn_models_are_the_models_each_turn_ran_on_in_order() -> None:
+    rollout: Final = "\n".join(
+        (
+            _line("turn_context", {"turn_id": "turn-1", "model": "first-model", "effort": "high"}),
+            _line("response_item", USER_PROMPT),
+            _line("compacted", REMOTE),
+            _line("turn_context", {"turn_id": "turn-2", "model": "second-model", "effort": "high"}),
+        )
+    )
+
+    assert rollout_turn_models(rollout) == ("first-model", "second-model")
+
+
 def test_the_harness_reads_only_the_rollout_of_the_thread_it_is_asked_about(tmp_path: Path) -> None:
     codex: Final = CodexHarness(
         home=tmp_path,