mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-06 07:48:06 +00:00
6254860560
* test(e2e): drive the real Codex CLI through the gateway and add the command round trip row Adds tests/e2e/codex: a harness that runs codex exec --json with a throwaway HOME and CODEX_HOME, a custom responses provider pointed at the gateway, the model list read from the gateway's /v1/models, the sandbox on and approvals off. Each test registers its setup's deployment via /model/new and calls it with a key scoped to it, both deleted on teardown. The first row checks that a value only a command can print comes back through the gateway on all five setups * test(e2e): type the Codex self-test event helper as a Mapping * test(e2e): clean up Codex test keys even when setup fails, and fail runs where Codex exits with an error * test(e2e): record a step when the Codex harness sets up its home and workspace The suite rules ask every public harness helper that does something to record a step, so a failed run shows whether it died while setting up Codex or while running it * Update run.py wip
23 lines
841 B
Python
23 lines
841 B
Python
from __future__ import annotations
|
|
|
|
from typing import Final
|
|
|
|
import pytest
|
|
from codex.harness import CodexHarness
|
|
from codex.setups import SETUPS, ModelSetup
|
|
from e2e_config import unique_marker
|
|
|
|
pytestmark = pytest.mark.e2e
|
|
|
|
|
|
@pytest.mark.parametrize("deployment", SETUPS)
|
|
class TestCodexCommandRoundTrip:
|
|
def test_answers_with_what_a_command_printed(self, codex: CodexHarness, setup: ModelSetup) -> None:
|
|
marker: Final = unique_marker()
|
|
workspace: Final = codex.new_workspace(files={"secret.txt": marker})
|
|
run: Final = codex.exec(
|
|
"Run `cat secret.txt`, then reply with exactly what it printed.", model=setup.model, cwd=workspace
|
|
)
|
|
assert run.exit_code == 0, run
|
|
assert any(marker in command.output for command in run.commands), run
|
|
assert marker in run.answer, run
|