diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index 7843c1a4091..5579c7cb1cd 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -50,7 +50,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke. `pr_gate_version_resolver.py` only forwards to `cli_harness/npm_version_resolver.py`, because the Buildkite runner images still run that path and install @latest without it - `cli_harness/` - no tests, only the plumbing every suite that drives a real coding-agent CLI shares: `npm_version_resolver.py` picks the CLI release to install (the newest npm release at least three days old, run by CI as `python tests/e2e/cli_harness/npm_version_resolver.py `, stdlib only), `lockdown.py` gives the CLI subprocess an allowlisted env and a fresh empty HOME so it never sees the runner's credentials or dotfiles, and `rate_limiter.py` is the cross-process token bucket per upstream that keeps parallel CLI runs under one request rate. Covered by `tests/e2e_harness/cli_harness/`. Code only one CLI suite needs stays in that suite -- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, and auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/` +- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted, and the round trip plus auto-compaction again with the deployment named after the real model, so Codex's `/v1/models` catalog gives it the provider's full entry instead of the generic one), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` with a key scoped to it, and the `codex` fixture points Codex at it. `real_name_setup` and `real_name_codex` do the same under the setup's `real_model_name` (`gpt-6.1-sol`, `claude-sonnet-5-5`) as a team model on a throwaway team with a team key, since LiteLLM pools deployments that share a name and parallel tests would otherwise share one. Everything is deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/` - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream, see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport. The Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`. Its specs follow the dashboard's rule against testing what shadcn and Base UI already guarantee, in `ui/litellm-dashboard/AGENTS.md` ## MCP suite: real Datadog only @@ -315,6 +315,8 @@ other... ``` ## Hard Rules +- no comments in code unless they are truly necessary + - no unit tests of a product feature under `tests/e2e`, and no mock tests or monkeypatching of code anywhere in it: a product feature is proven end to end against a live proxy, never with a unit test. if a contributor asks you to write an end to end test, do NOT stage a unit test with it. If you find a product gap, call it out in the PR description. the harness's own plumbing is tested outside the suite, in `tests/e2e_harness/` (mirroring this folder's layout), because the Buildkite e2e run copies `tests/e2e/` into the runner image and runs every test in it, so a harness test in here would count as a product test in the nightly numbers. those tests run without a proxy and take their inputs as arguments or env vars (setting an env var through pytest's `monkeypatch` fixture is fine, patching a function, class, or module is not), and no coverage-registry or compat-matrix cell rests on them. judge a change inside one of them by that standard, not as a misplaced product test - use model management endpoints to create new models for a test. this could be in a conftest / inline for each test. ask the user what they want. diff --git a/tests/e2e/codex/conftest.py b/tests/e2e/codex/conftest.py index 1ce6f2009a5..b6ab1768fa1 100644 --- a/tests/e2e/codex/conftest.py +++ b/tests/e2e/codex/conftest.py @@ -10,6 +10,7 @@ from codex.harness import CodexHarness, codex_harness from codex.setups import Deployment, ModelSetup from e2e_config import PROXY_BASE_URL, unique_marker from lifecycle import ResourceManager +from models import KeyGenerateBody, ModelInfoBody, ModelNewBody, TeamNewBody from proxy_client import ProxyClient @@ -28,15 +29,41 @@ def setup(deployment: Deployment, proxy: ProxyClient, resources: ResourceManager model: Final = f"e2e-codex-{unique_marker()}" model_id: Final = proxy.create_model(model, deployment.litellm_params, provider_live=True) resources.defer(lambda: proxy.delete_model(model_id)) - return ModelSetup(model=model, rate_bucket=deployment.rate_bucket) + return ModelSetup( + model=model, rate_bucket=deployment.rate_bucket, api_key=resources.key(models=[model], user_id=None) + ) @pytest.fixture -def codex(setup: ModelSetup, resources: ResourceManager) -> Iterator[CodexHarness]: +def real_name_setup(deployment: Deployment, proxy: ProxyClient, resources: ResourceManager) -> ModelSetup: + model: Final = deployment.real_model_name + team_id: Final = proxy.create_team(TeamNewBody(team_alias=f"e2e-codex-{unique_marker()}", models=[model])) + resources.defer(lambda: proxy.delete_team(team_id)) + api_key: Final = proxy.generate_key(KeyGenerateBody(team_id=team_id, models=[model])) + resources.defer(lambda: proxy.delete_key(api_key)) + model_id: Final = proxy.register_model( + ModelNewBody( + model_name=model, litellm_params=deployment.litellm_params, model_info=ModelInfoBody(team_id=team_id) + ), + listed_for=api_key, + provider_live=True, + ) + resources.defer(lambda: proxy.delete_model(model_id)) + return ModelSetup(model=model, rate_bucket=deployment.rate_bucket, api_key=api_key) + + +@pytest.fixture +def codex(setup: ModelSetup) -> Iterator[CodexHarness]: + yield from _codex_for(setup) + + +@pytest.fixture +def real_name_codex(real_name_setup: ModelSetup) -> Iterator[CodexHarness]: + yield from _codex_for(real_name_setup) + + +def _codex_for(setup: ModelSetup) -> Iterator[CodexHarness]: with codex_harness( - gateway=PROXY_BASE_URL, - api_key=resources.key(models=[setup.model], user_id=None), - rate_bucket=setup.rate_bucket, - limiter=get_default_limiter(), + gateway=PROXY_BASE_URL, api_key=setup.api_key, rate_bucket=setup.rate_bucket, limiter=get_default_limiter() ) as harness: yield harness diff --git a/tests/e2e/codex/setups.py b/tests/e2e/codex/setups.py index 4716706c51c..7e71e1d56bb 100644 --- a/tests/e2e/codex/setups.py +++ b/tests/e2e/codex/setups.py @@ -3,7 +3,7 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Final, Literal import pytest @@ -12,11 +12,13 @@ from cli_harness.rate_limiter import RateBucket from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from models import LiteLLMParamsBody -GPT_ON_BEDROCK_MANTLE: Final = "bedrock_mantle/openai.gpt-6.1-sol" -GPT_ON_BEDROCK_RUNTIME: Final = "bedrock/us.openai.gpt-6.1-sol" -GPT_ON_AZURE_OPENAI: Final = "azure/gpt-6.1-sol" -GPT_ON_OPENAI: Final = "openai/gpt-6.1-sol" -CLAUDE_ON_BEDROCK: Final = "bedrock/us.anthropic.claude-sonnet-5-5" +GPT: Final = "gpt-6.1-sol" +CLAUDE: Final = "claude-sonnet-5-5" +GPT_ON_BEDROCK_MANTLE: Final = f"bedrock_mantle/openai.{GPT}" +GPT_ON_BEDROCK_RUNTIME: Final = f"bedrock/us.openai.{GPT}" +GPT_ON_AZURE_OPENAI: Final = f"azure/{GPT}" +GPT_ON_OPENAI: Final = f"openai/{GPT}" +CLAUDE_ON_BEDROCK: Final = f"bedrock/us.anthropic.{CLAUDE}" _BEDROCK_REGION: Final = "us-east-1" type SetupId = Literal["bedrock_mantle", "bedrock_runtime", "azure_openai", "openai", "claude_bedrock"] @@ -26,12 +28,14 @@ type SetupId = Literal["bedrock_mantle", "bedrock_runtime", "azure_openai", "ope class Deployment: litellm_params: LiteLLMParamsBody rate_bucket: RateBucket + real_model_name: str @dataclass(frozen=True, slots=True) class ModelSetup: model: str rate_bucket: RateBucket + api_key: str = field(repr=False) def _drives(provider: Provider, model: str) -> pytest.MarkDecorator: @@ -49,7 +53,9 @@ def _drives(provider: Provider, model: str) -> pytest.MarkDecorator: SETUPS: Final = ( pytest.param( Deployment( - LiteLLMParamsBody(model=GPT_ON_BEDROCK_MANTLE, aws_region_name=_BEDROCK_REGION), RateBucket.BEDROCK_MANTLE + LiteLLMParamsBody(model=GPT_ON_BEDROCK_MANTLE, aws_region_name=_BEDROCK_REGION), + RateBucket.BEDROCK_MANTLE, + real_model_name=GPT, ), id="bedrock_mantle", marks=_drives(Provider.BEDROCK_MANTLE, GPT_ON_BEDROCK_MANTLE), @@ -58,6 +64,7 @@ SETUPS: Final = ( Deployment( LiteLLMParamsBody(model=GPT_ON_BEDROCK_RUNTIME, aws_region_name=_BEDROCK_REGION), RateBucket.BEDROCK_CONVERSE, + real_model_name=GPT, ), id="bedrock_runtime", marks=_drives(Provider.BEDROCK, GPT_ON_BEDROCK_RUNTIME), @@ -68,18 +75,25 @@ SETUPS: Final = ( model=GPT_ON_AZURE_OPENAI, api_base="os.environ/AZURE_API_BASE", api_key="os.environ/AZURE_API_KEY" ), RateBucket.AZURE_OPENAI, + real_model_name=GPT, ), id="azure_openai", marks=_drives(Provider.AZURE, GPT_ON_AZURE_OPENAI), ), pytest.param( - Deployment(LiteLLMParamsBody(model=GPT_ON_OPENAI, api_key="os.environ/OPENAI_API_KEY"), RateBucket.OPENAI), + Deployment( + LiteLLMParamsBody(model=GPT_ON_OPENAI, api_key="os.environ/OPENAI_API_KEY"), + RateBucket.OPENAI, + real_model_name=GPT, + ), id="openai", marks=_drives(Provider.OPENAI, GPT_ON_OPENAI), ), pytest.param( Deployment( - LiteLLMParamsBody(model=CLAUDE_ON_BEDROCK, aws_region_name=_BEDROCK_REGION), RateBucket.BEDROCK_CONVERSE + LiteLLMParamsBody(model=CLAUDE_ON_BEDROCK, aws_region_name=_BEDROCK_REGION), + RateBucket.BEDROCK_CONVERSE, + real_model_name=CLAUDE, ), id="claude_bedrock", marks=_drives(Provider.BEDROCK, CLAUDE_ON_BEDROCK), @@ -94,3 +108,10 @@ def with_expected_failure(setup_id: SetupId, *, reason: str) -> tuple[ParameterS else setup for setup in SETUPS ) + + +COMPACTION_SETUPS: Final = with_expected_failure( + "claude_bedrock", + reason="LiteLLM's Responses-to-chat bridge ignores Codex's compaction_trigger item, so Codex gets no" + " compaction back and fails the turn", +) diff --git a/tests/e2e/codex/test_auto_compaction_e2e.py b/tests/e2e/codex/test_auto_compaction_e2e.py index 5e0b2491002..13667c8bd6e 100644 --- a/tests/e2e/codex/test_auto_compaction_e2e.py +++ b/tests/e2e/codex/test_auto_compaction_e2e.py @@ -4,7 +4,7 @@ from typing import Final import pytest from codex.harness import CodexHarness -from codex.setups import ModelSetup, with_expected_failure +from codex.setups import COMPACTION_SETUPS, ModelSetup from e2e_config import unique_marker pytestmark = pytest.mark.e2e @@ -13,14 +13,7 @@ COMPACT_AFTER_TOKENS: Final = 500 PADDING: Final = "".join(f"line {number}: the quick brown fox jumps over the lazy dog\n" for number in range(400)) -@pytest.mark.parametrize( - "deployment", - with_expected_failure( - "claude_bedrock", - reason="LiteLLM's Responses-to-chat bridge ignores Codex's compaction_trigger item, so Codex gets no" - " compaction back and fails the turn", - ), -) +@pytest.mark.parametrize("deployment", COMPACTION_SETUPS) class TestCodexAutoCompaction: def test_remembers_a_value_after_the_gateway_compacts_the_session( self, codex: CodexHarness, setup: ModelSetup diff --git a/tests/e2e/codex/test_real_model_name_e2e.py b/tests/e2e/codex/test_real_model_name_e2e.py new file mode 100644 index 00000000000..7c4af4e9a4b --- /dev/null +++ b/tests/e2e/codex/test_real_model_name_e2e.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from codex.harness import CodexHarness +from codex.setups import COMPACTION_SETUPS, SETUPS, ModelSetup +from e2e_config import unique_marker + +pytestmark = pytest.mark.e2e + +COMPACT_AFTER_TOKENS: Final = 500 +PADDING: Final = "".join(f"line {number}: the quick brown fox jumps over the lazy dog\n" for number in range(400)) + + +class TestCodexUnderTheRealModelName: + @pytest.mark.parametrize("deployment", SETUPS) + def test_answers_with_what_a_command_printed( + self, real_name_codex: CodexHarness, real_name_setup: ModelSetup + ) -> None: + marker: Final = unique_marker() + workspace: Final = real_name_codex.new_workspace(files={"secret.txt": marker}) + run: Final = real_name_codex.exec( + "Run `cat secret.txt`, then reply with exactly what it printed.", model=real_name_setup.model, cwd=workspace + ) + assert run.exit_code == 0, run + assert any(marker in command.output for command in run.commands), run + assert marker in run.answer, run + + @pytest.mark.parametrize("deployment", COMPACTION_SETUPS) + def test_remembers_a_value_after_the_gateway_compacts_the_session( + self, real_name_codex: CodexHarness, real_name_setup: ModelSetup + ) -> None: + marker: Final = unique_marker() + workspace: Final = real_name_codex.new_workspace(files={"secret.txt": marker, "padding.txt": PADDING}) + first: Final = real_name_codex.exec( + "Run `cat secret.txt` and remember the value it prints. Then run `cat padding.txt`. Then reply with OK.", + model=real_name_setup.model, + cwd=workspace, + compact_after_tokens=COMPACT_AFTER_TOKENS, + ) + assert first.thread_id is not None, first + (workspace / "secret.txt").unlink() + second: Final = real_name_codex.resume( + first.thread_id, + "What value did secret.txt print earlier? Reply with just that value.", + model=real_name_setup.model, + cwd=workspace, + compact_after_tokens=COMPACT_AFTER_TOKENS, + ) + assert any(compaction.remote for compaction in real_name_codex.compactions(first.thread_id)), (first, second) + assert marker in second.answer, second