diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index 58876b15642..51d172f592f 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -50,7 +50,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke. `pr_gate_version_resolver.py` only forwards to `cli_harness/npm_version_resolver.py`, because the Buildkite runner images still run that path and install @latest without it - `cli_harness/` - no tests, only the plumbing every suite that drives a real coding-agent CLI shares: `npm_version_resolver.py` picks the CLI release to install (the newest npm release at least three days old, run by CI as `python tests/e2e/cli_harness/npm_version_resolver.py `, stdlib only), `lockdown.py` gives the CLI subprocess an allowlisted env and a fresh empty HOME so it never sees the runner's credentials or dotfiles, and `rate_limiter.py` is the cross-process token bucket per upstream that keeps parallel CLI runs under one request rate. Covered by `tests/e2e_harness/cli_harness/`. Code only one CLI suite needs stays in that suite -- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted, and an MCP tool, where Codex calls the tool of `mcp_server.py`, a stdio MCP server the test hands it with `add_mcp_server`, on a first and a resumed turn), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/` +- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted, and an MCP tool, where Codex calls the tool of `mcp_server.py`, a stdio MCP server the test hands it with `add_mcp_server`, on a first and a resumed turn), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the MCP tool calls with the text they returned, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/` - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream, see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport. The Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`. Its specs follow the dashboard's rule against testing what shadcn and Base UI already guarantee, in `ui/litellm-dashboard/AGENTS.md` ## MCP suite: real Datadog only diff --git a/tests/e2e/codex/harness.py b/tests/e2e/codex/harness.py index 04b96e5ae77..df005b5b5e7 100644 --- a/tests/e2e/codex/harness.py +++ b/tests/e2e/codex/harness.py @@ -106,9 +106,9 @@ class CodexHarness: def _mcp_server_table(name: str, server: McpServer) -> str: - env: Final = ", ".join(f"{key} = {json.dumps(value)}" for key, value in server.env.items()) + env: Final = ", ".join(f"{json.dumps(key)} = {json.dumps(value)}" for key, value in server.env.items()) return ( - f"\n[mcp_servers.{name}]\n" + f"\n[mcp_servers.{json.dumps(name)}]\n" f"command = {json.dumps(server.command)}\n" f"args = {json.dumps(server.args)}\n" f"env = {{ {env} }}\n" diff --git a/tests/e2e/codex/run.py b/tests/e2e/codex/run.py index 5195da67494..5ae3a429b89 100644 --- a/tests/e2e/codex/run.py +++ b/tests/e2e/codex/run.py @@ -54,6 +54,23 @@ class _FileChange(_Frozen): status: Literal["in_progress", "completed", "failed"] +class _McpContentBlock(_Frozen): + type: str + text: str | None = None + + +class _McpResult(_Frozen): + content: tuple[_McpContentBlock, ...] + + +class _McpToolCall(_Frozen): + type: Literal["mcp_tool_call"] + server: str + tool: str + result: _McpResult | None + status: Literal["in_progress", "completed", "failed"] + + class _ErrorItem(_Frozen): type: Literal["error"] message: str @@ -79,9 +96,12 @@ _Item = Annotated[ Annotated[_AgentMessage, Tag("agent_message")] | Annotated[_CommandExecution, Tag("command_execution")] | Annotated[_FileChange, Tag("file_change")] + | Annotated[_McpToolCall, Tag("mcp_tool_call")] | Annotated[_ErrorItem, Tag("error")] | Annotated[_Unmodelled, Tag("unmodelled")], - Discriminator(_known_or_unmodelled(frozenset({"agent_message", "command_execution", "file_change", "error"}))), + Discriminator( + _known_or_unmodelled(frozenset({"agent_message", "command_execution", "file_change", "mcp_tool_call", "error"})) + ), ] @@ -131,6 +151,13 @@ class Command: exit_code: int | None +@dataclass(frozen=True, slots=True) +class McpCall: + server: str + tool: str + output: str + + @dataclass(frozen=True, slots=True) class CodexError: message: str @@ -143,6 +170,7 @@ class CodexRun: thread_id: str | None commands: tuple[Command, ...] patched_files: tuple[Path, ...] + mcp_calls: tuple[McpCall, ...] answer: str usage: TokenUsage | None errors: tuple[CodexError, ...] @@ -164,6 +192,9 @@ def codex_run(*, exit_code: int, stdout: str, stderr: str) -> CodexRun: if isinstance(item, _CommandExecution) ), patched_files=tuple(Path(change.path) for change in chain.from_iterable(patch.changes for patch in patches)), + mcp_calls=tuple( + _mcp_call(item) for item in items if isinstance(item, _McpToolCall) and item.status == "completed" + ), answer=answers[-1] if answers else "", usage=usages[-1] if usages else None, errors=tuple(_errors(events)), @@ -171,6 +202,11 @@ def codex_run(*, exit_code: int, stdout: str, stderr: str) -> CodexRun: ) +def _mcp_call(item: _McpToolCall) -> McpCall: + blocks: Final = item.result.content if item.result else () + return McpCall(server=item.server, tool=item.tool, output="".join(block.text or "" for block in blocks)) + + def _errors(events: tuple[_Event, ...]) -> Iterator[CodexError]: for event in events: match event: diff --git a/tests/e2e/codex/test_mcp_tool_e2e.py b/tests/e2e/codex/test_mcp_tool_e2e.py index 44b76820dcf..41e4ecececa 100644 --- a/tests/e2e/codex/test_mcp_tool_e2e.py +++ b/tests/e2e/codex/test_mcp_tool_e2e.py @@ -18,10 +18,10 @@ MCP_SERVER: Final = Path(__file__).with_name("mcp_server.py") class TestCodexMcpTool: def test_calls_a_local_mcp_tool_again_on_a_resumed_turn(self, codex: CodexHarness, setup: ModelSetup) -> None: secret: Final = codex.home / "secret.txt" - codex.add_mcp_server( - "vault", McpServer(command=sys.executable, args=(str(MCP_SERVER),), env={"SECRET_FILE": str(secret)}) - ) + vault: Final = McpServer(command=sys.executable, args=(str(MCP_SERVER),), env={"SECRET_FILE": str(secret)}) + codex.add_mcp_server("vault", vault) workspace: Final = codex.new_workspace(files={}) + first_marker: Final = unique_marker() _ = secret.write_text(first_marker) first: Final = codex.exec( @@ -30,6 +30,7 @@ class TestCodexMcpTool: cwd=workspace, ) assert first.thread_id is not None, first + assert any(first_marker in call.output for call in first.mcp_calls), first assert first_marker in first.answer, first second_marker: Final = unique_marker() @@ -40,4 +41,5 @@ class TestCodexMcpTool: model=setup.model, cwd=workspace, ) + assert any(second_marker in call.output for call in second.mcp_calls), second assert second_marker in second.answer, second diff --git a/tests/e2e_harness/codex/test_codex_run.py b/tests/e2e_harness/codex/test_codex_run.py index 415967da114..93a6580444d 100644 --- a/tests/e2e_harness/codex/test_codex_run.py +++ b/tests/e2e_harness/codex/test_codex_run.py @@ -8,7 +8,7 @@ from collections.abc import Mapping from pathlib import Path from typing import Final -from codex.run import Command, codex_run +from codex.run import Command, McpCall, codex_run E2E_PYTEST_INI: Final = Path(__file__).resolve().parents[2] / "e2e" / "pytest.ini" @@ -60,6 +60,30 @@ class TestCodexRun: assert run.patched_files == (Path("/w/app.py"),) + def test_only_an_mcp_tool_call_that_completed_counts_and_keeps_the_text_the_tool_returned(self) -> None: + def call(status: str, result: Mapping[str, object] | None) -> str: + item: Final = { + "id": status, + "type": "mcp_tool_call", + "server": "vault", + "tool": "read_secret", + "arguments": {}, + "result": result, + "error": None if result else {"message": "tool call failed"}, + "status": status, + } + return _line({"type": "item.completed", "item": item}) + + returned: Final = { + "content": [{"type": "text", "text": "marker-7f3a91"}], + "structured_content": {"result": "marker-7f3a91"}, + } + run: Final = codex_run( + exit_code=0, stdout="\n".join((call("completed", returned), call("failed", None))), stderr="" + ) + + assert run.mcp_calls == (McpCall(server="vault", tool="read_secret", output="marker-7f3a91"),) + def test_a_turn_that_gave_up_on_a_5xx_is_rerun_by_the_suites_network_rule(self) -> None: failed: Final = _line( {"type": "turn.failed", "error": {"message": "exceeded retry limit, last status: 502 Bad Gateway"}}