diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index a4c51a53098..89bf24016da 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -50,7 +50,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke. `pr_gate_version_resolver.py` only forwards to `cli_harness/npm_version_resolver.py`, because the Buildkite runner images still run that path and install @latest without it - `cli_harness/` - no tests, only the plumbing every suite that drives a real coding-agent CLI shares: `npm_version_resolver.py` picks the CLI release to install (the newest npm release at least three days old, run by CI as `python tests/e2e/cli_harness/npm_version_resolver.py `, stdlib only), `lockdown.py` gives the CLI subprocess an allowlisted env and a fresh empty HOME so it never sees the runner's credentials or dotfiles, and `rate_limiter.py` is the cross-process token bucket per upstream that keeps parallel CLI runs under one request rate. Covered by `tests/e2e_harness/cli_harness/`. Code only one CLI suite needs stays in that suite -- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted, and an MCP tool, where Codex calls the tool of `mcp_server.py`, a stdio MCP server the test hands it with `add_mcp_server`, on a first and a resumed turn), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the MCP tool calls with the text they returned, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/` +- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted, and an MCP tool, where Codex calls the tool of `mcp_server.py`, a stdio MCP server the test hands it with `add_mcp_server`, on a first and a resumed turn, and a sub-agent, where Codex's multi-agent v2 hands a helper a random codeword as its task with shell tools off so the helper can only learn it from the task, and the helper's own final reply is read from its rollout), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the MCP tool calls with the text they returned, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `subagents` reads the sub-agents a thread spawned from their own rollouts, whose `session_meta` names that thread as their parent, each with its final replies and whether it forked the parent's conversation (`forked_from_id`), so the sub-agent row can reject a helper that saw the codeword in inherited history instead of its task. `exec` takes `enable` and `disable`, Codex feature names it passes as `--enable` and `--disable`. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/` - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream, see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport. The Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`. Its specs follow the dashboard's rule against testing what shadcn and Base UI already guarantee, in `ui/litellm-dashboard/AGENTS.md` ## MCP suite: real Datadog only diff --git a/tests/e2e/codex/harness.py b/tests/e2e/codex/harness.py index df005b5b5e7..cc29581f2ce 100644 --- a/tests/e2e/codex/harness.py +++ b/tests/e2e/codex/harness.py @@ -12,7 +12,7 @@ from typing import Final from cli_harness.lockdown import isolated_home, locked_down_env from cli_harness.rate_limiter import RateBucket, RateLimiter -from codex.rollout import Compaction, rollout_compactions +from codex.rollout import Compaction, Subagent, rollout_compactions, rollout_parent_thread, rollout_subagent from codex.run import CodexRun, codex_run from e2e_metadata import step @@ -71,8 +71,28 @@ class CodexHarness: _ = config.write(_mcp_server_table(name, server)) @step('Run Codex on {model} through the proxy with the prompt "{prompt}"') - def exec(self, prompt: str, *, model: str, cwd: Path, compact_after_tokens: int | None = None) -> CodexRun: - return self._run(("exec",), (*_compaction_flags(compact_after_tokens), "--model", model, "--", prompt), cwd) + def exec( + self, + prompt: str, + *, + model: str, + cwd: Path, + compact_after_tokens: int | None = None, + enable: tuple[str, ...] = (), + disable: tuple[str, ...] = (), + ) -> CodexRun: + return self._run( + ("exec",), + ( + *_compaction_flags(compact_after_tokens), + *_feature_flags(enable=enable, disable=disable), + "--model", + model, + "--", + prompt, + ), + cwd, + ) @step('Resume Codex thread {thread_id} on {model} through the proxy with the prompt "{prompt}"') def resume( @@ -88,6 +108,12 @@ class CodexHarness: rollouts: Final = sorted(self.codex_home.glob(f"sessions/**/rollout-*-{thread_id}.jsonl")) return tuple(chain.from_iterable(rollout_compactions(rollout.read_text()) for rollout in rollouts)) + def subagents(self, thread_id: str) -> tuple[Subagent, ...]: + rollouts: Final = tuple( + path.read_text() for path in sorted(self.codex_home.glob("sessions/**/rollout-*.jsonl")) + ) + return tuple(rollout_subagent(rollout) for rollout in rollouts if rollout_parent_thread(rollout) == thread_id) + def _run(self, subcommand: tuple[str, ...], args: tuple[str, ...], cwd: Path) -> CodexRun: _ = self.limiter.acquire(self.rate_bucket) completed: Final = subprocess.run( @@ -116,6 +142,13 @@ def _mcp_server_table(name: str, server: McpServer) -> str: ) +def _feature_flags(*, enable: tuple[str, ...], disable: tuple[str, ...]) -> tuple[str, ...]: + return ( + *chain.from_iterable(("--enable", feature) for feature in enable), + *chain.from_iterable(("--disable", feature) for feature in disable), + ) + + def _compaction_flags(after_tokens: int | None) -> tuple[str, ...]: match after_tokens: case None: diff --git a/tests/e2e/codex/rollout.py b/tests/e2e/codex/rollout.py index fcaeaa7ca42..5b8253fab10 100644 --- a/tests/e2e/codex/rollout.py +++ b/tests/e2e/codex/rollout.py @@ -1,19 +1,23 @@ -"""The compactions Codex recorded in a thread's rollout file, `CODEX_HOME/sessions/**/rollout-*-.jsonl`. +"""What Codex recorded in a thread's rollout file, `CODEX_HOME/sessions/**/rollout-*-.jsonl`. -`codex exec --json` prints no event for a compaction, so the rollout is the only place it shows. Each line is -`{"timestamp", "type", "payload"}` (`codex-rs/history/src/rollout_payload.rs`) and only `compacted` lines are read. +`codex exec --json` prints no event for a compaction or for a sub-agent's own turn, so the rollout is the only place +they show. Each line is `{"timestamp", "type", "payload"}` (`codex-rs/history/src/rollout_payload.rs`). A remote compaction is one whose replacement history carries the provider's `compaction` item, while a local one -replaces the history with a summary Codex wrote itself. +replaces the history with a summary Codex wrote itself. A sub-agent gets a rollout of its own whose `session_meta` +names the thread that spawned it, and carries `forked_from_id` when it inherited that thread's conversation. Each turn +it finishes ends in a `task_complete` event with its last message. """ from __future__ import annotations from dataclasses import dataclass -from typing import Final +from typing import Final, Literal from pydantic import BaseModel, ConfigDict _COMPACTED: Final = "compacted" +_SESSION_META: Final = "session_meta" +_EVENT: Final = "event_msg" _PROVIDER_COMPACTION_ITEM: Final = "compaction" @@ -34,14 +38,58 @@ class _CompactedPayload(_Frozen): replacement_history: tuple[_HistoryItem, ...] | None = None +class _SessionMeta(_Frozen): + parent_thread_id: str | None = None + forked_from_id: str | None = None + + +class _Event(_Frozen): + type: str + + +class _TaskComplete(_Frozen): + type: Literal["task_complete"] + last_agent_message: str | None = None + + @dataclass(frozen=True, slots=True) class Compaction: remote: bool +@dataclass(frozen=True, slots=True) +class Subagent: + forked: bool + replies: tuple[str, ...] + + def rollout_compactions(rollout: str) -> tuple[Compaction, ...]: - lines: Final = tuple(_Line.model_validate_json(line) for line in rollout.splitlines() if line.strip()) - return tuple(_compaction(line.payload) for line in lines if line.type == _COMPACTED) + return tuple(_compaction(line.payload) for line in _lines(rollout) if line.type == _COMPACTED) + + +def rollout_parent_thread(rollout: str) -> str | None: + return _session_meta(rollout).parent_thread_id + + +def rollout_subagent(rollout: str) -> Subagent: + return Subagent(forked=_session_meta(rollout).forked_from_id is not None, replies=rollout_final_messages(rollout)) + + +def rollout_final_messages(rollout: str) -> tuple[str, ...]: + events: Final = (line.payload for line in _lines(rollout) if line.type == _EVENT) + completions: Final = ( + _TaskComplete.model_validate(event) for event in events if _Event.model_validate(event).type == "task_complete" + ) + return tuple(done.last_agent_message for done in completions if done.last_agent_message is not None) + + +def _session_meta(rollout: str) -> _SessionMeta: + meta: Final = next((line.payload for line in _lines(rollout) if line.type == _SESSION_META), None) + return _SessionMeta.model_validate(meta) if meta is not None else _SessionMeta() + + +def _lines(rollout: str) -> tuple[_Line, ...]: + return tuple(_Line.model_validate_json(line) for line in rollout.splitlines() if line.strip()) def _compaction(payload: object) -> Compaction: diff --git a/tests/e2e/codex/test_subagent_e2e.py b/tests/e2e/codex/test_subagent_e2e.py new file mode 100644 index 00000000000..0ad86c8b49a --- /dev/null +++ b/tests/e2e/codex/test_subagent_e2e.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from codex.harness import CodexHarness +from codex.setups import SETUPS, ModelSetup +from e2e_config import unique_marker + +pytestmark = pytest.mark.e2e + + +@pytest.mark.parametrize("deployment", SETUPS) +class TestCodexSubagent: + def test_a_subagent_reads_the_task_its_parent_hands_it(self, codex: CodexHarness, setup: ModelSetup) -> None: + codeword: Final = unique_marker() + run: Final = codex.exec( + 'Spawn exactly one sub-agent with the spawn_agent tool, with fork_turns set to "none" so it does not' + f" inherit this conversation. Its task: reply with exactly this codeword and nothing else: {codeword}." + " Wait for its reply, then tell me the exact reply it sent you, verbatim.", + model=setup.model, + cwd=codex.new_workspace(files={}), + enable=("multi_agent_v2",), + disable=("shell_tool", "unified_exec"), + ) + assert run.exit_code == 0, run + assert run.thread_id is not None, run + subagents: Final = codex.subagents(run.thread_id) + assert subagents, run + assert not any(subagent.forked for subagent in subagents), (subagents, run) + assert any(any(codeword in reply for reply in subagent.replies) for subagent in subagents), (subagents, run) diff --git a/tests/e2e_harness/codex/test_codex_rollout.py b/tests/e2e_harness/codex/test_codex_rollout.py index 2ef27a8a9c5..3db39066204 100644 --- a/tests/e2e_harness/codex/test_codex_rollout.py +++ b/tests/e2e_harness/codex/test_codex_rollout.py @@ -8,7 +8,14 @@ from typing import Final from cli_harness.rate_limiter import RateBucket, RateLimiter from codex.harness import CodexHarness -from codex.rollout import Compaction, rollout_compactions +from codex.rollout import ( + Compaction, + Subagent, + rollout_compactions, + rollout_final_messages, + rollout_parent_thread, + rollout_subagent, +) THREAD_ID: Final = "01a18fea-b2e8-7ed2-a29b-1ce4d2b138c3" USER_PROMPT: Final = { @@ -60,13 +67,83 @@ class TestRolloutCompactions: assert rollout_compactions(_rollout()) == () -def test_the_harness_reads_only_the_rollout_of_the_thread_it_is_asked_about(tmp_path: Path) -> None: - codex: Final = CodexHarness( - home=tmp_path, +def _harness(home: Path) -> CodexHarness: + return CodexHarness( + home=home, api_key="unused", rate_bucket=RateBucket.OPENAI, - limiter=RateLimiter({}, tmp_path, time.monotonic, time.sleep), + limiter=RateLimiter({}, home, time.monotonic, time.sleep), ) + + +def _subagent_rollout( + *, thread_id: str, parent: str | None, final_messages: tuple[str | None, ...], forked: bool = False +) -> str: + meta: Final = ( + {"id": thread_id, "cli_version": "0.165.0"} + | ({"parent_thread_id": parent} if parent else {}) + | ({"forked_from_id": parent} if forked else {}) + ) + inherited: Final = (_line("session_meta", {"id": parent, "cli_version": "0.165.0"}),) if forked else () + return "\n".join( + ( + _line("session_meta", meta), + *inherited, + _line("event_msg", {"type": "task_started", "turn_id": "t1"}), + _line( + "response_item", + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "thinking"}]}, + ), + *( + _line("event_msg", {"type": "task_complete", "turn_id": "t1", "last_agent_message": message}) + for message in final_messages + ), + ) + ) + + +class TestRolloutSubagents: + def test_a_subagent_rollout_names_the_thread_that_spawned_it(self) -> None: + rollout: Final = _subagent_rollout(thread_id="child", parent=THREAD_ID, final_messages=()) + assert rollout_parent_thread(rollout) == THREAD_ID + + def test_a_top_level_rollout_has_no_parent(self) -> None: + assert rollout_parent_thread(_rollout()) is None + + def test_a_subagent_that_inherited_its_parents_conversation_is_forked(self) -> None: + rollout: Final = _subagent_rollout(thread_id="child", parent=THREAD_ID, final_messages=("hi",), forked=True) + assert rollout_subagent(rollout) == Subagent(forked=True, replies=("hi",)) + + def test_a_subagent_that_started_fresh_is_not_forked(self) -> None: + rollout: Final = _subagent_rollout(thread_id="child", parent=THREAD_ID, final_messages=("hi",)) + assert rollout_subagent(rollout) == Subagent(forked=False, replies=("hi",)) + + def test_the_final_messages_are_the_last_message_of_each_finished_turn(self) -> None: + rollout: Final = _subagent_rollout(thread_id="child", parent=THREAD_ID, final_messages=("first", None, "last")) + assert rollout_final_messages(rollout) == ("first", "last") + + +def test_the_harness_reads_only_the_subagents_of_the_thread_it_is_asked_about(tmp_path: Path) -> None: + codex: Final = _harness(tmp_path) + sessions: Final = codex.codex_home / "sessions" / "2026" / "11" / "06" + sessions.mkdir(parents=True) + rollouts: Final = { + THREAD_ID: _subagent_rollout(thread_id=THREAD_ID, parent=None, final_messages=("the parent's answer",)), + "child-a": _subagent_rollout(thread_id="child-a", parent=THREAD_ID, final_messages=("reply a",)), + "child-b": _subagent_rollout(thread_id="child-b", parent=THREAD_ID, final_messages=("reply b",), forked=True), + "stranger": _subagent_rollout(thread_id="stranger", parent="other-thread", final_messages=("not ours",)), + } + for thread_id, rollout in rollouts.items(): + _ = (sessions / f"rollout-2026-11-06T17-01-05-{thread_id}.jsonl").write_text(rollout) + + assert codex.subagents(THREAD_ID) == ( + Subagent(forked=False, replies=("reply a",)), + Subagent(forked=True, replies=("reply b",)), + ) + + +def test_the_harness_reads_only_the_rollout_of_the_thread_it_is_asked_about(tmp_path: Path) -> None: + codex: Final = _harness(tmp_path) sessions: Final = codex.codex_home / "sessions" / "2026" / "10" / "30" sessions.mkdir(parents=True) _ = (sessions / f"rollout-2026-10-30T20-03-32-{THREAD_ID}.jsonl").write_text(_rollout(REMOTE))