"""Pool ordering, reuse, and cleanup without Docker or model calls.""" # pylint: disable=missing-class-docstring,missing-function-docstring,protected-access,unused-argument from __future__ import annotations import asyncio import io import json import os from pathlib import Path import subprocess import tarfile import pytest from deploy.benchmark_docker import ( BenchmarkDockerPool, CasePlan, DockerReMeSandboxFactory, EvaluationQuery, JobRequest, SourceCandidate, SourceSnapshot, ) from deploy.benchmark_docker.docker_sandbox import SandboxCommandError, SandboxCommandTimeoutError, _archive_workspace from deploy.benchmark_docker import worker as sandbox_worker class FakeCase: def __init__(self, name: str, events: list[tuple]) -> None: self.name = name self.container_id = name self.events = events self.case_id = name self.reusable = True async def reset_case(self, case_id: str) -> None: self.case_id = case_id self.events.append((case_id, "reset")) async def upload_workspace(self, source: Path) -> None: assert source.exists() self.events.append((self.case_id, "upload", source.read_text(encoding="utf-8"))) async def run_build(self, jobs: list[JobRequest]) -> dict: self.events.append((self.case_id, "build", len(jobs))) await asyncio.sleep(0) return {"success": True} async def export_workspace(self, destination: Path) -> Path: destination = Path(destination) destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text(self.case_id, encoding="utf-8") self.events.append((self.case_id, "export_workspace")) return destination async def run_queries(self, queries: list[EvaluationQuery]) -> dict: self.events.append((self.case_id, "query", len(queries))) return {"success": True} async def export_evaluation(self, destination: Path) -> Path: self.events.append((self.case_id, "export_evaluation")) return destination async def export_build_log(self, destination: Path) -> Path: destination = Path(destination) destination.write_text("build failed", encoding="utf-8") self.events.append((self.case_id, "export_build_log")) return destination async def read_build_log(self) -> str: self.events.append((self.case_id, "read_build_log")) return "build failed" async def close(self) -> None: self.events.append((self.name, "close")) class FakeFactory: def __init__(self, events: list[tuple]) -> None: self.events = events async def create_cases(self, case_ids: list[str], *, concurrency: int) -> list[FakeCase]: assert len(case_ids) <= concurrency return [FakeCase(name, self.events) for name in case_ids] @pytest.mark.asyncio async def test_pool_builds_and_queries_each_case_then_reuses_workspace(tmp_path): events: list[tuple] = [] pool = BenchmarkDockerPool(FakeFactory(events), concurrency=1) # Put the dependent first to exercise task scheduling independently of input order. plans = [ CasePlan( "child", queries=(EvaluationQuery("q2", "question", "secret"),), reuse_workspace_from="parent", export_evaluation_to=tmp_path / "evaluation.tar.gz", ), CasePlan("parent", build_jobs=(JobRequest("build"),), queries=(EvaluationQuery("q1", "question", "secret"),)), ] results = await pool.run(plans) assert [result.case_id for result in results] == ["child", "parent"] assert all(result.success for result in results) assert ("child", "upload", "parent") in events assert ("child", "export_evaluation") in events assert events.index(("parent", "build", 1)) < events.index(("parent", "query", 1)) assert events.index(("parent", "export_workspace")) < events.index(("parent", "query", 1)) assert events.index(("parent", "query", 1)) < events.index(("child", "query", 1)) assert events[-1] == ("pool-0", "close") assert all(result.workspace_snapshot is None for result in results) @pytest.mark.asyncio async def test_pool_rejects_workspace_cycle_before_creating_containers(): events: list[tuple] = [] pool = BenchmarkDockerPool(FakeFactory(events), concurrency=2) with pytest.raises(ValueError, match="cycle"): await pool.run([CasePlan("a", reuse_workspace_from="b"), CasePlan("b", reuse_workspace_from="a")]) assert not events @pytest.mark.asyncio @pytest.mark.parametrize("raises", [False, True], ids=["failed-result", "exception"]) async def test_pool_keeps_build_log_on_failure(tmp_path, raises): class FailedBuildCase(FakeCase): async def run_build(self, jobs): if raises: raise TimeoutError("build exceeded command timeout") return {"success": False, "jobs": [{"error": "failed"}]} class FailedBuildFactory(FakeFactory): async def create_cases(self, case_ids, *, concurrency): return [FailedBuildCase(name, self.events) for name in case_ids] events: list[tuple] = [] log_path = tmp_path / "build.log" plan = CasePlan("failed", build_jobs=(JobRequest("build"),), export_build_log_to=log_path) result = (await BenchmarkDockerPool(FailedBuildFactory(events), concurrency=1).run([plan]))[0] assert not result.success if raises: assert result.error == "TimeoutError: build exceeded command timeout" assert result.build is None else: assert result.error == "build failed" assert result.build == {"success": False, "jobs": [{"error": "failed"}]} assert result.build_log == "build failed" assert log_path.read_text(encoding="utf-8") == "build failed" assert events.index(("failed", "read_build_log")) < events.index(("pool-0", "close")) @pytest.mark.asyncio async def test_pool_replacement_failure_preserves_healthy_cases(): events: list[tuple] = [] class Worker(FakeCase): async def run_build(self, jobs): if self.case_id == "broken": self.reusable = False raise RuntimeError("command failed") await asyncio.sleep(0.02) self.events.append((self.case_id, "finished")) return {"success": True} class Factory(FakeFactory): async def create_cases(self, case_ids, *, concurrency): return [Worker(name, self.events) for name in case_ids] async def create_case(self, case_id): raise RuntimeError("replacement failed") plans = [CasePlan(case_id, build_jobs=(JobRequest("build"),)) for case_id in ("broken", "healthy", "waiting")] results = await asyncio.wait_for(BenchmarkDockerPool(Factory(events), concurrency=2).run(plans), timeout=2) assert [result.case_id for result in results] == ["broken", "healthy", "waiting"] assert "command failed" in results[0].error assert "replacement failed" in results[0].error assert results[1].success and results[2].success assert ("healthy", "finished") in events and ("waiting", "finished") in events @pytest.mark.asyncio async def test_pool_exhaustion_unblocks_waiting_and_dependent_cases(): events: list[tuple] = [] class BrokenCase(FakeCase): async def run_build(self, jobs): self.reusable = False raise RuntimeError("command failed") class Factory(FakeFactory): async def create_cases(self, case_ids, *, concurrency): return [BrokenCase(name, self.events) for name in case_ids] async def create_case(self, case_id): raise RuntimeError("replacement failed") plans = [ CasePlan("dependent", reuse_workspace_from="waiting"), CasePlan("broken", build_jobs=(JobRequest("build"),)), CasePlan("waiting", build_jobs=(JobRequest("build"),)), CasePlan("waiting-2", build_jobs=(JobRequest("build"),)), ] results = await asyncio.wait_for(BenchmarkDockerPool(Factory(events), concurrency=1).run(plans), timeout=2) assert results[0].error == "source case 'waiting' failed" assert "replacement failed" in results[1].error assert all("no available workers" in result.error for result in results[2:]) assert not any(case_id.startswith("waiting") for case_id, action in events if action == "reset") @pytest.mark.asyncio async def test_pool_close_failure_retires_slot_without_replacement(): events: list[tuple] = [] class BrokenCase(FakeCase): async def run_build(self, jobs): self.reusable = False raise RuntimeError("command failed") async def close(self): self.events.append((self.name, "close failed")) raise RuntimeError("close failed") class Factory(FakeFactory): async def create_cases(self, case_ids, *, concurrency): return [BrokenCase(name, self.events) for name in case_ids] async def create_case(self, case_id): pytest.fail("replacement must not start while the old container may still be running") plans = [CasePlan(case_id, build_jobs=(JobRequest("build"),)) for case_id in ("broken", "next")] results = await asyncio.wait_for(BenchmarkDockerPool(Factory(events), concurrency=1).run(plans), timeout=2) assert "close failed" in results[0].error assert "no available workers" in results[1].error assert "close failed" in results[1].error assert ("next", "build", 1) not in events @pytest.mark.asyncio async def test_pool_cancels_case_before_closing_container(): started = asyncio.Event() events: list[tuple] = [] class BlockingCase(FakeCase): async def run_queries(self, queries): started.set() try: await asyncio.Event().wait() finally: self.events.append((self.case_id, "query_stopped")) class BlockingFactory(FakeFactory): async def create_cases(self, case_ids, *, concurrency): return [BlockingCase(name, self.events) for name in case_ids] plan = CasePlan("case", workspace_source=Path(__file__), queries=(EvaluationQuery("q", "question", "secret"),)) task = asyncio.create_task(BenchmarkDockerPool(BlockingFactory(events), concurrency=1).run([plan])) await asyncio.wait_for(started.wait(), timeout=2) task.cancel() with pytest.raises(asyncio.CancelledError): await task assert events.index(("case", "query_stopped")) < events.index(("pool-0", "close")) @pytest.mark.asyncio async def test_pool_final_close_failure_marks_only_cases_using_that_worker(): events: list[tuple] = [] class Worker(FakeCase): async def run_build(self, jobs): if self.case_id == "failed-build": return {"success": False} return await super().run_build(jobs) async def close(self): if self.name == "pool-0": raise RuntimeError("delete failed") await super().close() class Factory(FakeFactory): async def create_cases(self, case_ids, *, concurrency): return [Worker(name, self.events) for name in case_ids] plans = [ CasePlan(case_id, build_jobs=(JobRequest("build"),)) for case_id in ("first", "healthy", "failed-build", "last") ] results = await BenchmarkDockerPool(Factory(events), concurrency=2).run(plans) assert not results[0].success assert "pool-0" in results[0].error and "delete failed" in results[0].error assert results[1].success assert results[2].error.startswith("build failed;") assert "delete failed" in results[2].error assert results[3].success @pytest.mark.asyncio async def test_pool_reports_unused_worker_cleanup_failure(): events: list[tuple] = [] class Worker(FakeCase): async def run_build(self, jobs): return {"success": False} async def close(self): if self.name == "pool-1": raise RuntimeError("unused worker leaked") class Factory(FakeFactory): async def create_cases(self, case_ids, *, concurrency): return [Worker(name, self.events) for name in case_ids] plans = [CasePlan("parent", build_jobs=(JobRequest("build"),)), CasePlan("child", reuse_workspace_from="parent")] results = await BenchmarkDockerPool(Factory(events), concurrency=2).run(plans) assert results[0].error.startswith("build failed;") assert results[1].error.startswith("source case 'parent' failed;") assert all("unused worker leaked" in result.error for result in results) @pytest.mark.asyncio async def test_pool_finishes_final_cleanup_despite_repeated_cancellation(): started, release = asyncio.Event(), asyncio.Event() events: list[tuple] = [] class Worker(FakeCase): async def close(self): started.set() await release.wait() self.events.append((self.name, "closed")) raise RuntimeError("delete failed") class Factory(FakeFactory): async def create_cases(self, case_ids, *, concurrency): return [Worker(name, self.events) for name in case_ids] task = asyncio.create_task( BenchmarkDockerPool(Factory(events), concurrency=1).run([CasePlan("case", build_jobs=(JobRequest("build"),))]), ) await asyncio.wait_for(started.wait(), timeout=2) for _ in range(2): task.cancel() await asyncio.sleep(0) assert not task.done() release.set() with pytest.raises(asyncio.CancelledError) as failure: await asyncio.wait_for(task, timeout=2) assert events[-1] == ("pool-0", "closed") assert any("delete failed" in note for note in failure.value.__notes__) @pytest.mark.asyncio async def test_pool_does_not_replace_worker_for_cases_already_skipped(): events: list[tuple] = [] class Worker(FakeCase): async def run_build(self, jobs): if self.case_id == "parent": return {"success": False} # Let the dependent observe the failed parent before retirement. await asyncio.sleep(0) self.reusable = False raise RuntimeError("command timed out") class Factory(FakeFactory): async def create_cases(self, case_ids, *, concurrency): return [Worker(name, self.events) for name in case_ids] async def create_case(self, case_id): self.events.append((case_id, "replacement")) return FakeCase(case_id, self.events) plans = [ CasePlan("parent", build_jobs=(JobRequest("build"),)), CasePlan("skipped", reuse_workspace_from="parent"), CasePlan("last", build_jobs=(JobRequest("build"),)), ] results = await BenchmarkDockerPool(Factory(events), concurrency=1).run(plans) assert results[1].error == "source case 'parent' failed" assert "command timed out" in results[2].error assert not any(action == "replacement" for _, action in events) def test_source_snapshot_prunes_excluded_directories_and_secret_files(tmp_path, monkeypatch): (tmp_path / "reme").mkdir() (tmp_path / "reme" / "__init__.py").write_text("ok", encoding="utf-8") excluded = [tmp_path / name for name in ("baselines", ".venv", "node_modules")] for directory in excluded: (directory / "nested").mkdir(parents=True) (directory / "nested" / "large.json").write_text("private", encoding="utf-8") (tmp_path / ".env").write_text("secret", encoding="utf-8") scandir = os.scandir def checked_scandir(path): path = Path(path) assert not any(path == directory or directory in path.parents for directory in excluded) return scandir(path) monkeypatch.setattr(os, "scandir", checked_scandir) snapshot = SourceSnapshot.from_directory(tmp_path) assert snapshot.file_count == 1 def test_source_snapshot_preserves_custom_path_rules_and_symlink_checks(tmp_path): for name in ("pkg/cache/keep.py", "pkg/core.py", "pkg/cache/private.secret"): path = tmp_path / name path.parent.mkdir(parents=True, exist_ok=True) path.write_text(name, encoding="utf-8") (tmp_path / "pkg" / "ignored").symlink_to(tmp_path / "pkg" / "cache", target_is_directory=True) patterns = ("pkg/cache", "pkg/ignored", "pkg/*.secret") snapshot = SourceSnapshot.from_directory(tmp_path, excludes=patterns) with tarfile.open(fileobj=io.BytesIO(snapshot.archive), mode="r:gz") as archive: assert archive.getnames() == ["pkg/cache/keep.py", "pkg/core.py"] (tmp_path / "link").symlink_to(tmp_path / "pkg" / "cache", target_is_directory=True) with pytest.raises(ValueError, match="symbolic link: link"): SourceSnapshot.from_directory(tmp_path, excludes=patterns) def test_worker_merges_plugin_defaults_before_validating_config(tmp_path, monkeypatch): from reme.plugin import Plugin, PluginManager defaults = { "daily_dir": "notes", "digest_dir": "summaries", "jobs": {"plugin_version": {"backend": "base", "steps": [{"backend": "version_step"}]}}, } manager = PluginManager([Plugin(name="example", config=defaults)]) monkeypatch.setattr(PluginManager, "discover", classmethod(lambda cls, specs: manager)) # _prepare_runtime changes cwd and TMPDIR in the disposable worker process. monkeypatch.chdir(tmp_path) monkeypatch.setenv("TMPDIR", str(tmp_path)) config = tmp_path / "benchmark.yaml" config.write_text( "digest_dir: custom_digest\njobs:\n plugin_version:\n enable_serve: false\n", encoding="utf-8", ) case_root, prepared = sandbox_worker._prepare_runtime( { "case_root": str(tmp_path), "workspace_dir": str(tmp_path / "reme_workspace"), "config": str(config), "plugins": ["example"], }, ) assert prepared["daily_dir"] == "notes" assert prepared["digest_dir"] == "custom_digest" job = prepared["jobs"]["plugin_version"] assert job["backend"] == "base" assert job["steps"] == [{"backend": "version_step"}] assert job["enable_serve"] is False layout = json.loads((case_root / "runtime-layout.json").read_text(encoding="utf-8")) assert layout["configured_paths"]["daily_dir"] == "reme_workspace/notes" @pytest.mark.asyncio @pytest.mark.parametrize("mode", ["job", "queries"]) async def test_worker_keeps_startup_error_or_cancellation_when_close_fails(tmp_path, monkeypatch, mode): import reme.reme as reme_module primary = RuntimeError("startup exploded") if mode == "job" else asyncio.CancelledError("startup cancelled") closed = False class FakeReMe: def __init__(self, **_kwargs): pass async def start(self): raise primary async def close(self): nonlocal closed closed = True raise RuntimeError("close exploded") monkeypatch.setattr(reme_module, "ReMe", FakeReMe) monkeypatch.setattr(sandbox_worker, "_prepare_runtime", lambda _request: (tmp_path, {})) runner = sandbox_worker._run if mode == "job" else sandbox_worker._run_queries with pytest.raises(type(primary)) as failure: await runner({"job": "build", "queries": [{"query_id": "q1", "question": "Question?"}]}) assert failure.value is primary assert closed assert any("close exploded" in note for note in primary.__notes__) @pytest.mark.parametrize("mode", ["job", "build", "queries"]) @pytest.mark.parametrize("job_fails", [True, "result", False], ids=["exception", "failed-result", "close-only"]) def test_worker_response_preserves_failed_job_when_close_fails(tmp_path, monkeypatch, mode, job_fails): import reme.reme as reme_module from types import SimpleNamespace from reme.utils.counter import global_counter_inc class FakeReMe: def __init__(self, **_kwargs): self.context = SimpleNamespace(components={}, jobs={"build": object()}, metadata={}) async def start(self): pass async def run_job(self, job, **_kwargs): global_counter_inc(self.context.metadata, ["__job_counter", job]) if job_fails == "result": return SimpleNamespace(success=False, answer="job failed", metadata={"diagnostic": "original"}) if job_fails: raise RuntimeError("job exploded") return SimpleNamespace(success=True, answer="done", metadata={}) async def close(self): raise ValueError("close exploded") monkeypatch.setattr(reme_module, "ReMe", FakeReMe) monkeypatch.setattr(sandbox_worker, "_prepare_runtime", lambda _request: (tmp_path, {})) request = { "mode": mode, "case_root": str(tmp_path), "job": "build", "jobs": [{"job": "build"}], "queries": [{"query_id": "q1", "question": "Question?", "answer_job": "build"}], } request_path = tmp_path / "request.json" response_path = tmp_path / "response.json" request_path.write_text(json.dumps(request), encoding="utf-8") monkeypatch.setattr("sys.argv", ["worker", "--request", str(request_path), "--response", str(response_path)]) assert sandbox_worker.main() == 1 response = json.loads(response_path.read_text(encoding="utf-8")) assert response["success"] is False if not job_fails: assert response["error"] == "ValueError: close exploded" return result = response if mode == "build": result = response["jobs"][0]["result"] log = (tmp_path / "build_log" / "build.log").read_text(encoding="utf-8") assert ("job failed" if job_fails == "result" else "job exploded") in log assert "close exploded" in log elif mode == "queries": query_result = response["queries"][0] saved = json.loads((tmp_path / "queries" / "q1" / "result.json").read_text(encoding="utf-8")) assert saved == query_result result = query_result["answer_result"] if job_fails == "result": assert result["error"] == "job failed" assert result["metadata"] == {"diagnostic": "original"} else: assert result["error"] == "RuntimeError: job exploded" assert result["exception"]["type"] == "RuntimeError" assert "job exploded" in result["traceback"] assert result["job_call_counts"] == {"build": 1} assert result["token_usage"] == {} diagnostics = query_result["cleanup_errors"] if mode == "queries" else result["cleanup_errors"] assert diagnostics[0]["error"] == "ValueError: close exploded" assert diagnostics[0]["exception"]["type"] == "ValueError" assert "close exploded" in diagnostics[0]["traceback"] @pytest.mark.asyncio @pytest.mark.parametrize("failure_phase", ["config", "application"]) async def test_build_log_captures_initialization_failure(tmp_path, monkeypatch, failure_phase): if failure_phase == "config": def fail_config(_request): raise ValueError("invalid benchmark config") monkeypatch.setattr(sandbox_worker, "_prepare_runtime", fail_config) expected = "ValueError: invalid benchmark config" else: config = { "workspace_dir": str(tmp_path / "workspace"), "service": {"backend": "cli"}, "jobs": {"broken": {"backend": "missing"}}, "enable_logo": False, "log_to_console": False, "log_to_file": False, } monkeypatch.setattr(sandbox_worker, "_prepare_runtime", lambda _request: (tmp_path, config)) expected = "ValueError: Unregistered backend 'missing' for Job 'broken'" with pytest.raises(ValueError, match="invalid benchmark config|Unregistered backend"): await sandbox_worker._run_build({"case_root": str(tmp_path), "jobs": [{"job": "build"}]}) log_text = (tmp_path / "build_log" / "build.log").read_text(encoding="utf-8") assert "Traceback (most recent call last):" in log_text assert expected in log_text @pytest.mark.asyncio @pytest.mark.parametrize("failure_phase", [None, "start", "checkpoint", "close", "start_and_close", "job"]) async def test_build_log_captures_runtime_failures_and_preserves_primary_error(tmp_path, monkeypatch, failure_phase): import reme.reme as reme_module from reme.utils import get_logger monkeypatch.setattr(sandbox_worker, "_prepare_runtime", lambda _request: (tmp_path, {})) startup_error = RuntimeError("startup exploded") checkpoint_error = RuntimeError("checkpoint exploded") close_error = RuntimeError("close exploded") closed = False class FakeReMe: def __init__(self, **_kwargs): get_logger(log_to_console=False, log_to_file=False, force_init=True) async def start(self): if failure_phase in {"start", "start_and_close"}: raise startup_error async def close(self): nonlocal closed closed = True if failure_phase in {"close", "start_and_close"}: raise close_error async def run_job(_app, _job, _arguments): if failure_phase == "job": return {"success": False, "error": "RuntimeError: job exploded", "traceback": "job traceback"} return {"success": True} def checkpoint(*_args): raise checkpoint_error monkeypatch.setattr(reme_module, "ReMe", FakeReMe) monkeypatch.setattr(sandbox_worker, "_run_job_on_app", run_job) monkeypatch.setattr(sandbox_worker, "_commit_memory_checkpoint", checkpoint) jobs = [{"job": "build"}] if failure_phase == "checkpoint": jobs[0]["memory_checkpoint"] = "boundary" request = {"case_root": str(tmp_path), "jobs": jobs} if failure_phase in {"start", "checkpoint", "close", "start_and_close"}: expected_error = checkpoint_error if failure_phase == "checkpoint" else startup_error if failure_phase == "close": expected_error = close_error with pytest.raises(RuntimeError) as failure: await sandbox_worker._run_build(request) assert failure.value is expected_error else: result = await sandbox_worker._run_build(request) assert result["success"] is (failure_phase is None) assert closed get_logger().info("after build") log_text = (tmp_path / "build_log" / "build.log").read_text(encoding="utf-8") assert "[sandbox] build phase started" in log_text assert "[sandbox] build phase finished" in log_text assert "after build" not in log_text if failure_phase in {"start", "checkpoint", "close", "start_and_close"}: assert "Traceback (most recent call last):" in log_text assert str(expected_error) in log_text if failure_phase == "start_and_close": assert "close exploded" in log_text assert any("close exploded" in note for note in failure.value.__notes__) if failure_phase == "job": assert "RuntimeError: job exploded" in log_text assert "job traceback" in log_text @pytest.mark.asyncio async def test_worker_clears_search_budget_between_queries(tmp_path, monkeypatch): import reme.reme as reme_module from reme.utils import get_logger monkeypatch.setattr(sandbox_worker, "_prepare_runtime", lambda _request: (tmp_path, {})) seen_metadata: list[dict] = [] apps = [] class FakeReMe: def __init__(self, **_kwargs): from types import SimpleNamespace self.context = SimpleNamespace(metadata={"shared": "retained"}) apps.append(self) async def start(self): pass async def close(self): pass async def run_job(app, _job, _arguments): get_logger().info(f"answer body: {_arguments['query']}") seen_metadata.append(dict(app.context.metadata)) app.context.metadata["tool_contexts"] = {"same-id": {"seen": True}} app.context.metadata["__search_call_budgets"] = {"same-id": {"count": 1}} return {"success": True, "answer": "ok", "metadata": {}, "error": None} monkeypatch.setattr(reme_module, "ReMe", FakeReMe) monkeypatch.setattr(sandbox_worker, "_run_job_on_app", run_job) result = await sandbox_worker._run_queries( { "case_root": str(tmp_path), "queries": [ {"query_id": "q1", "question": "first"}, {"query_id": "q2", "question": "second"}, ], }, ) assert result["success"] assert len(apps) == 1 assert seen_metadata == [{"shared": "retained"}, {"shared": "retained"}] assert apps[0].context.metadata == {"shared": "retained"} for query_id, own, other in (("q1", "first", "second"), ("q2", "second", "first")): log = (tmp_path / "queries" / query_id / "answer.log").read_text(encoding="utf-8") assert f"answer body: {own}" in log assert f"answer body: {other}" not in log @pytest.mark.asyncio async def test_worker_checkpoint_commits_only_memory_paths(tmp_path, monkeypatch): workspace = tmp_path / "workspace" workspace.mkdir() subprocess.run(["git", "init", "--quiet", str(workspace)], check=True) (workspace / "daily").mkdir() (workspace / "daily" / "note.md").write_text("memory", encoding="utf-8") (workspace / "digest").mkdir() (workspace / "digest" / "note.md").write_text("digest", encoding="utf-8") (workspace / "sessions").mkdir() (workspace / "sessions" / "note.md").write_text("session", encoding="utf-8") (workspace / "other.txt").write_text("unrelated", encoding="utf-8") config = {"workspace_dir": str(workspace), "daily_dir": "daily", "digest_dir": "digest", "session_dir": "sessions"} monkeypatch.setattr(sandbox_worker, "_prepare_runtime", lambda _request: (tmp_path, config)) result = await sandbox_worker._run_checkpoint({"message": "memory boundary"}) assert result["success"] is True assert result["memory_checkpoint"]["message"] == "memory boundary" committed = subprocess.run( ["git", "show", "--pretty=format:", "--name-only", "HEAD"], cwd=workspace, capture_output=True, text=True, check=True, ) assert set(committed.stdout.splitlines()) == {"daily/note.md", "digest/note.md", "sessions/note.md"} assert result["memory_checkpoint"]["paths"] == ["daily", "digest", "sessions"] second = await sandbox_worker._run_checkpoint({"message": "empty boundary"}) assert second["memory_checkpoint"]["commit_sha"] != result["memory_checkpoint"]["commit_sha"] config["daily_dir"] = "../outside" with pytest.raises(ValueError, match="escapes the runtime workspace"): await sandbox_worker._run_checkpoint({"message": "unsafe boundary"}) class FakeBackend: def __init__(self) -> None: self.files: dict[str, bytes] = {} self.commands: list[list[str]] = [] self.worker_responses: dict[str, dict] = {} async def write_file(self, path: str, data: bytes) -> None: self.files[path] = data async def read_file(self, path: str) -> bytes: try: return self.files[path] except KeyError as exc: raise FileNotFoundError(path) from exc async def exec_shell(self, command: list[str], *, cwd=None, timeout=None): from types import SimpleNamespace self.commands.append(command) output = b"" exit_code = 0 if command[:2] == ["test", "-d"]: exit_code = 0 if any(path.startswith(command[2] + "/") for path in self.files) else 1 if len(command) > 2 and command[1] == "-c": output = json.dumps( { "reme": "/workspace/candidate/reme/__init__.py", "agentscope": "2.0.9", "plugins": {"lme": "/workspace/candidate/plugins/lme/src/reme_lme/__init__.py"}, }, ).encode() if "/workspace/harness/worker.py" in command: request = json.loads(self.files[command[command.index("--request") + 1]]) mode = request.get("mode", "job") default = {"success": True, "jobs": []} if mode == "build" else {"success": True} response = self.worker_responses.get(mode, default) self.files[command[command.index("--response") + 1]] = json.dumps(response).encode() exit_code = 0 if response["success"] else 1 if command[:2] == ["tar", "-czf"]: self.files[command[2]] = b"workspace archive" return SimpleNamespace(exit_code=exit_code, stdout=output, stderr=b"", ok=lambda: exit_code == 0) class TimedOutBackend(FakeBackend): async def exec_shell(self, command: list[str], *, cwd=None, timeout=None): if "/workspace/harness/worker.py" in command: from types import SimpleNamespace self.commands.append(command) self.files["/workspace/case/build_log/build.log"] = b"build still running" return SimpleNamespace(exit_code=-1, stdout=b"", stderr=b"timed out", ok=lambda: False) return await super().exec_shell(command, cwd=cwd, timeout=timeout) class FakeWorkspace: def __init__(self, backend: FakeBackend) -> None: self.backend = backend self.closed = False self.stopped = False async def initialize(self) -> None: pass def get_backend(self) -> FakeBackend: return self.backend async def close(self) -> None: self.closed = True async def stop(self) -> None: self.stopped = True def _fake_docker_factory(tmp_path, workspace_builder, *, install_projects=(".",)): (tmp_path / "reme").mkdir() (tmp_path / "reme" / "__init__.py").write_text("ok", encoding="utf-8") return DockerReMeSandboxFactory( SourceCandidate(SourceSnapshot.from_directory(tmp_path), install_projects=install_projects), plugins=("lme",), workspace_builder=workspace_builder, ) @pytest.mark.asyncio async def test_backend_timeout_prevents_direct_case_reset(tmp_path): workspace = FakeWorkspace(TimedOutBackend()) factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: workspace) case = await factory.create_case("timed-out") try: with pytest.raises(SandboxCommandTimeoutError): await case.run_build([JobRequest("build")]) assert not case.reusable assert workspace.stopped and workspace.closed assert case.actions[-1].exit_code == -1 assert await case.read_build_log() == "build still running" with pytest.raises(SandboxCommandError, match="no longer reusable"): await case.reset_case("next") finally: await case.close() assert workspace.closed @pytest.mark.asyncio async def test_external_timeout_prevents_case_reset(tmp_path): started = asyncio.Event() class BlockingBackend(FakeBackend): async def exec_shell(self, command, *, cwd=None, timeout=None): if "/workspace/harness/worker.py" in command: self.commands.append(command) started.set() await asyncio.Event().wait() return await super().exec_shell(command, cwd=cwd, timeout=timeout) workspace = FakeWorkspace(BlockingBackend()) factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: workspace) case = await factory.create_case("timed-out") try: build = asyncio.create_task(case.run_build([JobRequest("build")])) await asyncio.wait_for(started.wait(), timeout=2) with pytest.raises(TimeoutError): await asyncio.wait_for(build, timeout=0.01) assert not case.reusable assert workspace.stopped and workspace.closed with pytest.raises(SandboxCommandError, match="no longer reusable"): await case.reset_case("next") inbox_count = sum("/inbox/" in path for path in workspace.backend.files) with pytest.raises(SandboxCommandError, match="no longer reusable"): await case.run_build([JobRequest("build")]) assert sum("/inbox/" in path for path in workspace.backend.files) == inbox_count finally: await case.close() assert workspace.closed @pytest.mark.asyncio @pytest.mark.parametrize("failure", ["deadline", "backend-timeout", "cancel"]) async def test_interrupted_worker_stops_before_reading_logs_and_removal(tmp_path, failure): events = [] started = asyncio.Event() stopping = asyncio.Event() release_stop = asyncio.Event() class BlockingBackend(FakeBackend): async def exec_shell(self, command, *, cwd=None, timeout=None): if "/workspace/harness/worker.py" in command: self.files["/workspace/case/build_log/build.log"] = b"closing application" started.set() if failure == "backend-timeout": raise TimeoutError("backend timed out") await asyncio.Event().wait() # Includes a worker stuck in Application.close(). return await super().exec_shell(command, cwd=cwd, timeout=timeout) async def read_file(self, path): events.append("read") assert workspace.stopped return await super().read_file(path) class OrderedWorkspace(FakeWorkspace): async def stop(self): stopping.set() await release_stop.wait() events.append("stop") await super().stop() async def close(self): events.append("close") await super().close() workspace = OrderedWorkspace(BlockingBackend()) factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: workspace) case = await factory.create_case("interrupted") case.command_timeout = 0.01 if failure == "deadline" else 10 task = asyncio.create_task(case.run_build([JobRequest("build")])) await asyncio.wait_for(started.wait(), timeout=2) if failure == "cancel": task.cancel() await asyncio.wait_for(stopping.wait(), timeout=2) if failure == "cancel": task.cancel() # Repeated cancellation must not interrupt container cleanup. release_stop.set() expected = asyncio.CancelledError if failure == "cancel" else SandboxCommandTimeoutError with pytest.raises(expected): await asyncio.wait_for(task, timeout=2) assert events == ["stop", "read", "close"] assert workspace.closed and not case.reusable assert await case.read_build_log() == "closing application" await case.close() assert events == ["stop", "read", "close"] @pytest.mark.asyncio async def test_failed_stop_still_removes_container_and_preserves_timeout(tmp_path): class BrokenStopWorkspace(FakeWorkspace): async def stop(self): raise RuntimeError("kill failed") workspace = BrokenStopWorkspace(TimedOutBackend()) factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: workspace) case = await factory.create_case("failed-stop") with pytest.raises(SandboxCommandTimeoutError) as failure: await case.run_build([JobRequest("build")]) assert workspace.closed assert any("kill failed" in note for note in failure.value.__notes__) with pytest.raises(SandboxCommandError, match="log unavailable"): await case.read_build_log() @pytest.mark.asyncio async def test_blocked_diagnostics_cannot_prevent_removal(tmp_path, monkeypatch): from deploy.benchmark_docker import docker_sandbox monkeypatch.setattr(docker_sandbox, "_LOG_TIMEOUT", 0.01) class BlockedLogBackend(TimedOutBackend): async def read_file(self, path): assert workspace.stopped await asyncio.Event().wait() workspace = FakeWorkspace(BlockedLogBackend()) factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: workspace) case = await factory.create_case("blocked-log") with pytest.raises(SandboxCommandTimeoutError): await asyncio.wait_for(case.run_build([JobRequest("build")]), timeout=2) assert workspace.closed @pytest.mark.asyncio async def test_cancelled_request_upload_prevents_case_reset(tmp_path): started = asyncio.Event() class BlockingBackend(FakeBackend): async def write_file(self, path, data): if "/inbox/" in path: started.set() await asyncio.Event().wait() await super().write_file(path, data) workspace = FakeWorkspace(BlockingBackend()) factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: workspace) case = await factory.create_case("cancelled-upload") try: build = asyncio.create_task(case.run_build([JobRequest("build")])) await asyncio.wait_for(started.wait(), timeout=2) build.cancel() with pytest.raises(asyncio.CancelledError): await build assert not case.reusable with pytest.raises(SandboxCommandError, match="no longer reusable"): await case.reset_case("next") finally: await case.close() assert workspace.closed @pytest.mark.asyncio @pytest.mark.parametrize("operation", ["write", "exec"]) async def test_backend_error_prevents_case_reset(tmp_path, operation): class FailingBackend(FakeBackend): async def write_file(self, path, data): if operation == "write" and "/inbox/" in path: raise RuntimeError("Docker stream failed") await super().write_file(path, data) async def exec_shell(self, command, *, cwd=None, timeout=None): if operation == "exec" and "/workspace/harness/worker.py" in command: raise RuntimeError("Docker stream failed") return await super().exec_shell(command, cwd=cwd, timeout=timeout) workspace = FakeWorkspace(FailingBackend()) factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: workspace) case = await factory.create_case(f"{operation}-error") try: with pytest.raises(RuntimeError, match="Docker stream failed"): await case.run_build([JobRequest("build")]) assert not case.reusable with pytest.raises(SandboxCommandError, match="no longer reusable"): await case.reset_case("next") finally: await case.close() assert workspace.closed @pytest.mark.asyncio async def test_cancelled_close_can_be_retried(tmp_path): started = asyncio.Event() class InterruptibleWorkspace(FakeWorkspace): def __init__(self): super().__init__(FakeBackend()) self.close_calls = 0 async def close(self): self.close_calls += 1 if self.close_calls == 1: started.set() await asyncio.Event().wait() self.closed = True workspace = InterruptibleWorkspace() factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: workspace) case = await factory.create_case("close-retry") closing = asyncio.create_task(case.close()) await asyncio.wait_for(started.wait(), timeout=2) closing.cancel() with pytest.raises(asyncio.CancelledError): await closing assert not case.reusable assert not workspace.closed await case.close() assert workspace.closed assert workspace.close_calls == 2 @pytest.mark.asyncio async def test_source_candidate_installs_plugin_and_closes_container(tmp_path): backend = FakeBackend() workspace = FakeWorkspace(backend) factory = _fake_docker_factory( tmp_path, lambda _id, _image, _env: workspace, install_projects=(".", "plugins/lme"), ) case = await factory.create_case("example") try: assert any(command[-1] == "/workspace/candidate/plugins/lme" for command in backend.commands) assert "/workspace/candidate.tar.gz" in backend.files assert "/workspace/harness/worker.py" in backend.files assert (await case.run_build([JobRequest("auto_memory")]))["success"] exported = await case.export_workspace(tmp_path / "workspace.tar.gz") assert exported.read_bytes() == b"workspace archive" with pytest.raises(ValueError, match="private"): await case.run_queries( [EvaluationQuery("q1", "question", "secret", answer_arguments={"golden_answer": "secret"})], ) finally: await case.close() assert workspace.closed @pytest.mark.asyncio async def test_job_and_checkpoint_share_worker_request_path(tmp_path): backend = FakeBackend() workspace = FakeWorkspace(backend) factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: workspace) case = await factory.create_case("example") try: result = await case.run_job("answer", {"query": "hello"}) assert result.success assert case.actions[-1].name == "job:answer" job_request = json.loads(backend.files[next(path for path in backend.files if "/inbox/" in path)]) assert job_request["mode"] == "job" assert job_request["arguments"] == {"query": "hello"} await case.commit_memory_history("memory boundary") assert case.actions[-1].name == "worker:checkpoint" requests = [json.loads(content) for path, content in backend.files.items() if "/inbox/" in path] assert requests[-1]["mode"] == "checkpoint" assert requests[-1]["message"] == "memory boundary" assert not any(command[0] == "git" and "commit" in command for command in backend.commands) with pytest.raises(ValueError, match="non-empty"): await case.commit_memory_history(" ") backend.worker_responses["checkpoint"] = {"success": False, "error": "git commit failed"} with pytest.raises(SandboxCommandError, match="git commit failed"): await case.commit_memory_history("next boundary") assert case.actions[-1].exit_code == 1 finally: await case.close() @pytest.mark.asyncio async def test_pool_replaces_worker_after_backend_timeout(tmp_path): timed_out_workspace = FakeWorkspace(TimedOutBackend()) replacement_workspace = FakeWorkspace(FakeBackend()) workspaces = iter((timed_out_workspace, replacement_workspace)) factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: next(workspaces)) plans = [ CasePlan("timed-out", build_jobs=(JobRequest("build"),), export_build_log_to=tmp_path / "build.log"), CasePlan("next", build_jobs=(JobRequest("build"),)), ] results = await BenchmarkDockerPool(factory, concurrency=1).run(plans) assert "SandboxCommandTimeoutError" in results[0].error assert results[0].build_log == "build still running" assert (tmp_path / "build.log").read_text(encoding="utf-8") == "build still running" assert results[1].success assert timed_out_workspace.closed and replacement_workspace.closed assert sum(command[:2] == ["rm", "-rf"] for command in timed_out_workspace.backend.commands) == 1 @pytest.mark.asyncio @pytest.mark.parametrize( ("method", "required", "description", "members"), [ ( "export_memory_construction", "build_log/build.log", "memory construction export", ["manifest.json", "reme_workspace", "build_log"], ), ("export_queries", "queries", "query export", ["manifest.json", "queries"]), ("export_query", "queries/q1", "query export", ["manifest.json", "queries/q1"]), ( "export_evaluation", "queries", "evaluation export", ["manifest.json", "reme_workspace", "queries"], ), ], ) async def test_archive_exports_check_required_artifacts(tmp_path, method, required, description, members): backend = FakeBackend() case = await _fake_docker_factory(tmp_path, lambda _id, _image, _env: FakeWorkspace(backend)).create_case("example") destination = tmp_path / "exports" / "result.tar.gz" export = getattr(case, method) args = ("q1", destination) if method == "export_query" else (destination,) try: with pytest.raises(SandboxCommandError) as failure: await export(*args) assert str(failure.value) == f"{description} is missing required artifact: /workspace/case/{required}" artifact = "queries/q1/answer.log" if required.startswith("queries") else required backend.files[f"/workspace/case/{artifact}"] = b"{}" assert await export(*args) == destination assert ["tar", "-czf", "/tmp/reme-case-export.tar.gz", "-C", "/workspace/case", *members] in backend.commands finally: await case.close() @pytest.mark.asyncio @pytest.mark.parametrize( "method", ["export_workspace", "export_memory_construction", "export_evaluation"], ) async def test_exported_memory_can_seed_queries_without_build(tmp_path, method): class ArchiveBackend(FakeBackend): async def exec_shell(self, command, *, cwd=None, timeout=None): result = await super().exec_shell(command, cwd=cwd, timeout=timeout) if command[:2] == ["tar", "-czf"]: root = command[command.index("-C") + 1] + "/" members = command[command.index("-C") + 2 :] stream = io.BytesIO() with tarfile.open(fileobj=stream, mode="w:gz") as archive: for path, content in self.files.items(): if not path.startswith(root): continue name = path[len(root) :] if not any( member == "." or name == member or name.startswith(member + "/") for member in members ): continue info = tarfile.TarInfo(name) info.size = len(content) archive.addfile(info, io.BytesIO(content)) self.files[command[2]] = stream.getvalue() return result source_backend, query_backend = ArchiveBackend(), FakeBackend() workspaces = iter((FakeWorkspace(source_backend), FakeWorkspace(query_backend))) factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: next(workspaces)) source = await factory.create_case("build") query_case = await factory.create_case("query") try: assert (await source.run_build([JobRequest("build")]))["success"] source_backend.files.update( { "/workspace/case/reme_workspace/daily/note.md": b"saved memory", "/workspace/case/reme_workspace/.git/HEAD": b"ref: refs/heads/main\n", "/workspace/case/build_log/build.log": b"build log", "/workspace/case/queries/summary.json": b"{}", }, ) saved = await getattr(source, method)(tmp_path / "memory.tar.gz") await query_case.upload_workspace(saved) uploaded = query_backend.files["/tmp/reme-workspace-upload.tar.gz"] with tarfile.open(fileobj=io.BytesIO(uploaded), mode="r:gz") as archive: assert set(archive.getnames()) == {"daily/note.md", ".git/HEAD"} assert archive.extractfile("daily/note.md").read() == b"saved memory" assert (await query_case.run_queries([EvaluationQuery("q1", "Question?", "Answer")]))["success"] requests = [json.loads(content) for path, content in query_backend.files.items() if "/inbox/" in path] assert [request["mode"] for request in requests] == ["queries"] finally: await source.close() await query_case.close() @pytest.mark.parametrize( ("manifest", "workspace_path", "error"), [ (None, "reme_workspace/daily/note.md", "unmarked case archive"), ({"archive_format": "reme-case-v1", "workspace_root": None}, None, "no reusable workspace"), ({"archive_format": "reme-case-v1", "workspace_root": "../escape"}, None, "workspace root"), ({"archive_format": "reme-case-v2", "workspace_root": "reme_workspace"}, None, "unsupported.*format"), ({"archive_format": "reme-case-v1", "workspace_root": "reme_workspace"}, None, "workspace is missing"), ({}, "reme_workspace/daily/note.md", None), ], ) def test_workspace_import_checks_case_archive_format(tmp_path, manifest, workspace_path, error): source = tmp_path / "case.tar.gz" files = {"queries/q1/result.json": b"{}"} if manifest is not None: files["manifest.json"] = json.dumps(manifest).encode() if workspace_path is not None: files[workspace_path] = b"memory" with tarfile.open(source, "w:gz") as archive: for name, content in files.items(): info = tarfile.TarInfo(name) info.size = len(content) archive.addfile(info, io.BytesIO(content)) if error is not None: with pytest.raises(ValueError, match=error): _archive_workspace(source) else: with tarfile.open(fileobj=io.BytesIO(_archive_workspace(source)), mode="r:gz") as archive: assert archive.getnames() == ["daily/note.md"] @pytest.mark.asyncio @pytest.mark.parametrize("query_state", ["batch", "single", "partial", "absent"]) async def test_default_analysis_export_saves_logs_outside_archive(tmp_path, query_state): class AnalysisBackend(FakeBackend): async def exec_shell(self, command, *, cwd=None, timeout=None): result = await super().exec_shell(command, cwd=cwd, timeout=timeout) if command[:2] == ["tar", "-czf"] and command[-1] == "queries": stream = io.BytesIO() with tarfile.open(fileobj=stream, mode="w:gz") as archive: for path, content in self.files.items(): if not path.startswith("/workspace/case/queries/"): continue name = path.removeprefix("/workspace/case/") info = tarfile.TarInfo(name) info.size = len(content) archive.addfile(info, io.BytesIO(content)) self.files[command[2]] = stream.getvalue() return result backend = AnalysisBackend() workspace = FakeWorkspace(backend) factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: workspace) case = await factory.create_case("example") try: backend.files["/workspace/case/runtime-layout.json"] = ( b'{"workspace_root":"reme_workspace","analysis_excludes":[]}' ) backend.files["/workspace/case/build_log/build.log"] = b"build log" if query_state == "single": backend.worker_responses["query"] = {"success": True, "queries": [{"answer": "ok"}]} assert await case.run_query({"query_id": "q1", "question": "Question?"}) == {"answer": "ok"} if query_state != "absent": backend.files["/workspace/case/queries/q1/answer.log"] = b"answer log" if query_state in {"batch", "single"}: backend.files["/workspace/case/queries/q1/result.json"] = b'{"answer":"ok"}' if query_state == "batch": backend.files["/workspace/case/queries/summary.json"] = b"{}" archive = tmp_path / "analysis.tar.gz" assert await case.export(archive) == archive assert archive.read_bytes() == b"workspace archive" artifacts = tmp_path / "analysis.tar.gz.artifacts" assert (artifacts / "build_log" / "build.log").read_bytes() == b"build log" query_files = { path.removeprefix("/workspace/case/"): content for path, content in backend.files.items() if path.startswith("/workspace/case/queries/") } assert { str(path.relative_to(artifacts)): path.read_bytes() for path in (artifacts / "queries").rglob("*") if path.is_file() } == query_files query_commands = [command for command in backend.commands if command[:2] == ["tar", "-czf"]] assert len(query_commands) == (1 if query_state == "absent" else 2) archive_command = next( command for command in backend.commands if command[:3] == ["tar", "-czf", "/tmp/reme-case-export.tar.gz"] ) assert "build_log" not in archive_command assert "queries" not in archive_command finally: await case.close() @pytest.mark.asyncio async def test_cancelled_case_initialization_closes_workspace(tmp_path): started = asyncio.Event() class BlockingWorkspace(FakeWorkspace): async def initialize(self): started.set() await asyncio.Event().wait() workspace = BlockingWorkspace(FakeBackend()) factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: workspace) task = asyncio.create_task(factory.create_case("cancelled")) await asyncio.wait_for(started.wait(), timeout=2) task.cancel() with pytest.raises(asyncio.CancelledError): await task assert workspace.closed @pytest.mark.asyncio async def test_cancelled_batch_creation_closes_completed_workspace_and_reports_cleanup_failure(tmp_path): second_started = asyncio.Event() class BlockingWorkspace(FakeWorkspace): async def initialize(self): second_started.set() await asyncio.Event().wait() async def close(self): await super().close() raise RuntimeError("delete failed") first = FakeWorkspace(FakeBackend()) second = BlockingWorkspace(FakeBackend()) workspaces = iter((first, second)) factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: next(workspaces)) task = asyncio.create_task(factory.create_cases(["first", "second"], concurrency=2)) await asyncio.wait_for(second_started.wait(), timeout=2) await asyncio.sleep(0) task.cancel() with pytest.raises(asyncio.CancelledError) as failure: await task assert first.closed and second.closed assert any("delete failed" in note for note in failure.value.__notes__) @pytest.mark.asyncio async def test_failed_case_creation_preserves_initialization_error_when_close_fails(tmp_path): class BrokenWorkspace(FakeWorkspace): async def initialize(self): raise RuntimeError("initialization failed") async def close(self): raise RuntimeError("close failed") factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: BrokenWorkspace(FakeBackend())) with pytest.raises(RuntimeError, match="initialization failed") as failure: await factory.create_case("broken") assert any("close failed" in note for note in failure.value.__notes__) @pytest.mark.asyncio async def test_failed_batch_creation_preserves_creation_error_when_close_fails(tmp_path): class CloseFailsWorkspace(FakeWorkspace): async def close(self): raise RuntimeError("close failed") class InitializeFailsWorkspace(FakeWorkspace): async def initialize(self): raise RuntimeError("initialization failed") workspaces = iter((CloseFailsWorkspace(FakeBackend()), InitializeFailsWorkspace(FakeBackend()))) factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: next(workspaces)) with pytest.raises(RuntimeError, match="initialization failed") as failure: await factory.create_cases(["first", "second"], concurrency=2) assert any("close failed" in note for note in failure.value.__notes__) @pytest.mark.asyncio async def test_factory_finishes_failed_creation_cleanup_before_cancellation(tmp_path): started, release = asyncio.Event(), asyncio.Event() class BrokenWorkspace(FakeWorkspace): async def initialize(self): raise RuntimeError("initialization failed") async def close(self): started.set() await release.wait() self.closed = True workspace = BrokenWorkspace(FakeBackend()) factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: workspace) task = asyncio.create_task(factory.create_case("failed")) await asyncio.wait_for(started.wait(), timeout=2) for _ in range(2): task.cancel() await asyncio.sleep(0) assert not task.done() release.set() with pytest.raises(asyncio.CancelledError) as failure: await asyncio.wait_for(task, timeout=2) assert workspace.closed assert "initialization failed" in str(failure.value.__cause__) @pytest.mark.asyncio async def test_batch_reports_all_initialization_and_cleanup_failures(tmp_path): class BrokenWorkspace(FakeWorkspace): async def initialize(self): raise RuntimeError("initialization failed") async def close(self): raise RuntimeError("delete failed") factory = _fake_docker_factory(tmp_path, lambda _id, _image, _env: BrokenWorkspace(FakeBackend())) with pytest.raises(RuntimeError, match="initialization failed") as failure: await factory.create_cases(["first", "second"], concurrency=2) assert any("additional creation failure" in note for note in failure.value.__notes__) assert sum("delete failed" in note for note in failure.value.__notes__) == 2 @pytest.mark.asyncio async def test_pool_retries_strict_delete_without_replacing_uncertain_container(tmp_path): from types import SimpleNamespace from unittest.mock import AsyncMock pytest.importorskip("aiodocker") from deploy.benchmark_docker.direct_docker_workspace import DirectDockerWorkspace workspaces = [] container = SimpleNamespace( kill=AsyncMock(), wait=AsyncMock(), delete=AsyncMock(side_effect=[RuntimeError("delete failed"), RuntimeError("delete failed"), None]), ) client = SimpleNamespace(close=AsyncMock()) class StrictWorkspace(DirectDockerWorkspace): # Provisioning deliberately populates the inherited Docker handles. # pylint: disable=attribute-defined-outside-init async def _provision_backend(self): self._container, self._client, self._backend = container, client, TimedOutBackend() def build_workspace(workspace_id, image, env): workspace = StrictWorkspace(workspace_id=workspace_id, base_image=image, env=env) workspaces.append(workspace) return workspace factory = _fake_docker_factory(tmp_path, build_workspace) plans = [CasePlan(case_id, build_jobs=(JobRequest("build"),)) for case_id in ("timed-out", "waiting")] results = await asyncio.wait_for(BenchmarkDockerPool(factory, concurrency=1).run(plans), timeout=2) assert not any(result.success for result in results) assert "worker close failed" in results[0].error assert "delete failed" in results[1].error assert len(workspaces) == 1 assert container.delete.await_count == 3 assert workspaces[0]._container is workspaces[0]._client is None client.close.assert_awaited_once()