mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
test(e2e): make the Codex long session prove every earlier command output reached the model
This commit is contained in:
parent
cf71fbaeaa
commit
306d5ab5f6
1 changed files with 13 additions and 6 deletions
|
|
@ -12,16 +12,23 @@ pytestmark = pytest.mark.e2e
|
|||
|
||||
@pytest.mark.parametrize("deployment", SETUPS)
|
||||
class TestCodexLongSession:
|
||||
def test_follows_a_chain_of_files_one_command_at_a_time(self, codex: CodexHarness, setup: ModelSetup) -> None:
|
||||
def test_remembers_every_command_output_in_a_chain_of_files(self, codex: CodexHarness, setup: ModelSetup) -> None:
|
||||
codes: Final = (unique_marker(), unique_marker(), unique_marker())
|
||||
second: Final = f"{unique_marker()}.txt"
|
||||
third: Final = f"{unique_marker()}.txt"
|
||||
marker: Final = f"DONE-{unique_marker()}"
|
||||
workspace: Final = codex.new_workspace(files={"start.txt": second, second: third, third: marker})
|
||||
workspace: Final = codex.new_workspace(
|
||||
files={
|
||||
"start.txt": f"code {codes[0]}\nnext {second}\n",
|
||||
second: f"code {codes[1]}\nnext {third}\n",
|
||||
third: f"code {codes[2]}\nlast file\n",
|
||||
}
|
||||
)
|
||||
run: Final = codex.exec(
|
||||
"Run `cat start.txt`. It prints a file name. Run `cat` on that file, one command per step, and keep"
|
||||
" following the file names until a command prints a value that starts with DONE-. Reply with that value.",
|
||||
"Run `cat start.txt`. Each file prints a code and then the name of the next file, until one says last"
|
||||
" file. Run `cat` on each next file, one command per step. Reply with the three codes in the order you"
|
||||
" read them.",
|
||||
model=setup.model,
|
||||
cwd=workspace,
|
||||
)
|
||||
assert len(run.commands) >= 3, run
|
||||
assert marker in run.answer, run
|
||||
assert all(code in run.answer for code in codes), run
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue