diff --git a/deploy/benchmark_docker/Dockerfile.candidate.dockerignore b/deploy/benchmark_docker/Dockerfile.candidate.dockerignore index c9628b66..27ede6f8 100644 --- a/deploy/benchmark_docker/Dockerfile.candidate.dockerignore +++ b/deploy/benchmark_docker/Dockerfile.candidate.dockerignore @@ -1,21 +1,52 @@ +# Only candidate-side benchmark plugins are needed in this image. +# Add another explicit plugin directory here before selecting it with REME_PLUGIN_PATH. +plugins/* +!plugins/lme +!plugins/beam + +# Keep exclusions after plugin exceptions so nested secrets stay excluded. .git -.reme -.env -.env.* -.venv +**/.DS_Store +**/.ssh +**/.codex +**/.claude +**/private* +**/.env +**/.env.* +**/.reme +**/.venv +**/venv **/__pycache__ +**/*.py[cod] **/.pytest_cache +**/.mypy_cache +**/.ruff_cache +**/.coverage* +**/htmlcov **/node_modules **/dist -**/build +**/dist-static +**/.generated +**/.next +**/.vite +**/.wrangler +**/.cache **/*.egg-info +**/build +**/logs +**/*.log +**/*.tmp +reme_studio/src/reme_studio/static benchmark -baselines -meta-reme -output -tests +cookbook docs github-pages integrations skills +tests + +# Candidate build inputs do not need local benchmark results or Studio sources. +baselines +meta-reme +output reme_studio diff --git a/deploy/benchmark_docker/docker_sandbox.py b/deploy/benchmark_docker/docker_sandbox.py index 7b3c48ce..ea360213 100644 --- a/deploy/benchmark_docker/docker_sandbox.py +++ b/deploy/benchmark_docker/docker_sandbox.py @@ -693,7 +693,16 @@ class DockerReMeSandbox: serialized: list[dict[str, Any]] = [] query_ids: list[str] = [] for query in queries: - value = asdict(query) if isinstance(query, EvaluationQuery) else dict(query) + value = ( + { + "query_id": query.query_id, + "question": query.question, + "answer_arguments": query.answer_arguments, + "answer_job": query.answer_job, + } + if isinstance(query, EvaluationQuery) + else dict(query) + ) query_id = value.get("query_id") if not isinstance(query_id, str): raise ValueError("every evaluation query requires a string query_id") @@ -916,7 +925,7 @@ class DockerReMeSandbox: The copy is made through a validated archive, so each sandbox receives an independent filesystem tree and can run queries in parallel. """ - archive = _archive_workspace(source) + archive = await asyncio.to_thread(_archive_workspace, source) async with self._operation_lock: assert self.backend is not None, "sandbox is not initialized" if clear: diff --git a/tests/unit/test_benchmark_docker_queries.py b/tests/unit/test_benchmark_docker_queries.py new file mode 100644 index 00000000..0b87951a --- /dev/null +++ b/tests/unit/test_benchmark_docker_queries.py @@ -0,0 +1,65 @@ +"""Host-only query answers never participate in candidate serialization.""" + +# pylint: disable=missing-function-docstring,protected-access + +import json +import threading +from unittest.mock import AsyncMock + +import pytest + +from deploy.benchmark_docker import EvaluationQuery, ImageCandidate +from deploy.benchmark_docker.docker_sandbox import DockerReMeSandbox + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["run_query", "run_queries"]) +@pytest.mark.parametrize("as_mapping", [False, True]) +async def test_query_upload_ignores_uncopyable_golden_answer(monkeypatch, method, as_mapping): + case = DockerReMeSandbox( + case_id="query", + container_id="unused", + candidate=ImageCandidate("unused"), + workspace=AsyncMock(), + config="default", + plugins=(), + command_timeout=5, + environment_names=[], + ) + arguments = {"limit": 3} + golden_answer = threading.Lock() + query = EvaluationQuery("q1", "Question?", golden_answer, arguments, "custom_answer") + if as_mapping: + query = { + "query_id": "q1", + "question": "Question?", + "golden_answer": golden_answer, + "answer_arguments": arguments, + "answer_job": "custom_answer", + "rubric": golden_answer, + } + result = {"answer": "candidate answer"} + worker = AsyncMock(return_value={"queries": [result]}) + monkeypatch.setattr(case, "_run_worker_request", worker) + + if method == "run_query": + assert await case.run_query(query) == result + else: + assert await case.run_queries([query]) == {"queries": [result]} + + mode, payload = worker.await_args.args + assert mode == ("query" if method == "run_query" else "queries") + assert payload == { + "queries": [ + { + "query_id": "q1", + "question": "Question?", + "answer_arguments": {"limit": 3}, + "answer_job": "custom_answer", + }, + ], + } + assert payload["queries"][0]["answer_arguments"] is not arguments + # A host-only lock is neither deep-copied nor included in the JSON request. + json.dumps(payload) + assert not golden_answer.locked() diff --git a/tests/unit/test_benchmark_docker_upload.py b/tests/unit/test_benchmark_docker_upload.py new file mode 100644 index 00000000..92157dd5 --- /dev/null +++ b/tests/unit/test_benchmark_docker_upload.py @@ -0,0 +1,95 @@ +"""Candidate build exclusions and nonblocking workspace upload without Docker.""" + +# pylint: disable=missing-function-docstring,protected-access + +import asyncio +import io +from pathlib import Path +import tarfile +import threading +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from deploy.benchmark_docker import ImageCandidate +from deploy.benchmark_docker import docker_sandbox as sandbox_module +from deploy.benchmark_docker.docker_sandbox import _archive_workspace + + +def test_candidate_image_preserves_root_exclusions_after_plugin_exceptions(): + root = Path(__file__).resolve().parents[2] + root_rules = (root / ".dockerignore").read_text(encoding="utf-8").splitlines() + candidate_rules = ( + (root / "deploy/benchmark_docker/Dockerfile.candidate.dockerignore").read_text(encoding="utf-8").splitlines() + ) + exceptions = [rule for rule in candidate_rules if rule.startswith("!")] + assert set(exceptions) == {"!plugins/lme", "!plugins/beam"} + assert "plugins/*" in candidate_rules + last_exception = max(candidate_rules.index(rule) for rule in exceptions) + exclusions = candidate_rules[last_exception + 1 :] + # New root protections must also cover the Dockerfile-specific context, + # including secrets inside otherwise allowed plugin directories. + assert set(root_rules) - {"plugins", ""} <= set(exclusions) + assert {"baselines", "meta-reme", "output", "reme_studio"} <= set(exclusions) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("archive_error", [False, True]) +async def test_workspace_archiving_keeps_event_loop_responsive(tmp_path, monkeypatch, archive_error): + backend = AsyncMock() + backend.exec_shell.return_value = SimpleNamespace(exit_code=0, stdout=b"", stderr=b"", ok=lambda: True) + case = sandbox_module.DockerReMeSandbox( + case_id="archive-thread", + container_id="unused", + candidate=ImageCandidate("unused"), + workspace=AsyncMock(), + config="default", + plugins=(), + command_timeout=5, + environment_names=[], + ) + case.backend = backend + source = tmp_path / "memory" + source.mkdir() + (source / "note.md").write_text("saved memory", encoding="utf-8") + loop = asyncio.get_running_loop() + started = asyncio.Event() + release = threading.Event() + event_loop_thread = threading.get_ident() + + def blocking_archive(path): + assert threading.get_ident() != event_loop_thread + loop.call_soon_threadsafe(started.set) + if not release.wait(timeout=5): + raise TimeoutError("test did not release archive worker") + if archive_error: + raise ValueError("invalid workspace archive") + return _archive_workspace(path) + + monkeypatch.setattr(sandbox_module, "_archive_workspace", blocking_archive) + upload = asyncio.create_task(case.upload_workspace(source)) + try: + await asyncio.wait_for(started.wait(), timeout=2) + await asyncio.sleep(0) + assert not upload.done() + # No workspace clearing or upload may happen before validation finishes. + backend.exec_shell.assert_not_awaited() + backend.write_file.assert_not_awaited() + release.set() + if archive_error: + with pytest.raises(ValueError, match="invalid workspace archive"): + await upload + backend.exec_shell.assert_not_awaited() + backend.write_file.assert_not_awaited() + else: + await upload + backend.write_file.assert_awaited_once() + archive_path, payload = backend.write_file.await_args.args + assert archive_path == sandbox_module.WORKSPACE_ARCHIVE + with tarfile.open(fileobj=io.BytesIO(payload), mode="r:gz") as archive: + assert archive.extractfile("note.md").read() == b"saved memory" + finally: + release.set() + await asyncio.gather(upload, return_exceptions=True) + await case.close()