Fix candidate exclusions and benchmark upload serialization

deploy/benchmark_docker/Dockerfile.candidate.dockerignore

@@ -1,21 +1,52 @@
+# Only candidate-side benchmark plugins are needed in this image.
+# Add another explicit plugin directory here before selecting it with REME_PLUGIN_PATH.
+plugins/*
+!plugins/lme
+!plugins/beam
+
+# Keep exclusions after plugin exceptions so nested secrets stay excluded.
 .git
-.reme
-.env
-.env.*
-.venv
+**/.DS_Store
+**/.ssh
+**/.codex
+**/.claude
+**/private*
+**/.env
+**/.env.*
+**/.reme
+**/.venv
+**/venv
 **/__pycache__
+**/*.py[cod]
 **/.pytest_cache
+**/.mypy_cache
+**/.ruff_cache
+**/.coverage*
+**/htmlcov
 **/node_modules
 **/dist
-**/build
+**/dist-static
+**/.generated
+**/.next
+**/.vite
+**/.wrangler
+**/.cache
 **/*.egg-info
+**/build
+**/logs
+**/*.log
+**/*.tmp
+reme_studio/src/reme_studio/static
 benchmark
-baselines
-meta-reme
-output
-tests
+cookbook
 docs
 github-pages
 integrations
 skills
+tests
+
+# Candidate build inputs do not need local benchmark results or Studio sources.
+baselines
+meta-reme
+output
 reme_studio

deploy/benchmark_docker/docker_sandbox.py

@@ -693,7 +693,16 @@ class DockerReMeSandbox:
         serialized: list[dict[str, Any]] = []
         query_ids: list[str] = []
         for query in queries:
-            value = asdict(query) if isinstance(query, EvaluationQuery) else dict(query)
+            value = (
+                {
+                    "query_id": query.query_id,
+                    "question": query.question,
+                    "answer_arguments": query.answer_arguments,
+                    "answer_job": query.answer_job,
+                }
+                if isinstance(query, EvaluationQuery)
+                else dict(query)
+            )
             query_id = value.get("query_id")
             if not isinstance(query_id, str):
                 raise ValueError("every evaluation query requires a string query_id")
@@ -916,7 +925,7 @@ class DockerReMeSandbox:
         The copy is made through a validated archive, so each sandbox receives
         an independent filesystem tree and can run queries in parallel.
         """
-        archive = _archive_workspace(source)
+        archive = await asyncio.to_thread(_archive_workspace, source)
         async with self._operation_lock:
             assert self.backend is not None, "sandbox is not initialized"
             if clear:

tests/unit/test_benchmark_docker_queries.py

@@ -0,0 +1,65 @@
+"""Host-only query answers never participate in candidate serialization."""
+
+# pylint: disable=missing-function-docstring,protected-access
+
+import json
+import threading
+from unittest.mock import AsyncMock
+
+import pytest
+
+from deploy.benchmark_docker import EvaluationQuery, ImageCandidate
+from deploy.benchmark_docker.docker_sandbox import DockerReMeSandbox
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("method", ["run_query", "run_queries"])
+@pytest.mark.parametrize("as_mapping", [False, True])
+async def test_query_upload_ignores_uncopyable_golden_answer(monkeypatch, method, as_mapping):
+    case = DockerReMeSandbox(
+        case_id="query",
+        container_id="unused",
+        candidate=ImageCandidate("unused"),
+        workspace=AsyncMock(),
+        config="default",
+        plugins=(),
+        command_timeout=5,
+        environment_names=[],
+    )
+    arguments = {"limit": 3}
+    golden_answer = threading.Lock()
+    query = EvaluationQuery("q1", "Question?", golden_answer, arguments, "custom_answer")
+    if as_mapping:
+        query = {
+            "query_id": "q1",
+            "question": "Question?",
+            "golden_answer": golden_answer,
+            "answer_arguments": arguments,
+            "answer_job": "custom_answer",
+            "rubric": golden_answer,
+        }
+    result = {"answer": "candidate answer"}
+    worker = AsyncMock(return_value={"queries": [result]})
+    monkeypatch.setattr(case, "_run_worker_request", worker)
+
+    if method == "run_query":
+        assert await case.run_query(query) == result
+    else:
+        assert await case.run_queries([query]) == {"queries": [result]}
+
+    mode, payload = worker.await_args.args
+    assert mode == ("query" if method == "run_query" else "queries")
+    assert payload == {
+        "queries": [
+            {
+                "query_id": "q1",
+                "question": "Question?",
+                "answer_arguments": {"limit": 3},
+                "answer_job": "custom_answer",
+            },
+        ],
+    }
+    assert payload["queries"][0]["answer_arguments"] is not arguments
+    # A host-only lock is neither deep-copied nor included in the JSON request.
+    json.dumps(payload)
+    assert not golden_answer.locked()

tests/unit/test_benchmark_docker_upload.py

@@ -0,0 +1,95 @@
+"""Candidate build exclusions and nonblocking workspace upload without Docker."""
+
+# pylint: disable=missing-function-docstring,protected-access
+
+import asyncio
+import io
+from pathlib import Path
+import tarfile
+import threading
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from deploy.benchmark_docker import ImageCandidate
+from deploy.benchmark_docker import docker_sandbox as sandbox_module
+from deploy.benchmark_docker.docker_sandbox import _archive_workspace
+
+
+def test_candidate_image_preserves_root_exclusions_after_plugin_exceptions():
+    root = Path(__file__).resolve().parents[2]
+    root_rules = (root / ".dockerignore").read_text(encoding="utf-8").splitlines()
+    candidate_rules = (
+        (root / "deploy/benchmark_docker/Dockerfile.candidate.dockerignore").read_text(encoding="utf-8").splitlines()
+    )
+    exceptions = [rule for rule in candidate_rules if rule.startswith("!")]
+    assert set(exceptions) == {"!plugins/lme", "!plugins/beam"}
+    assert "plugins/*" in candidate_rules
+    last_exception = max(candidate_rules.index(rule) for rule in exceptions)
+    exclusions = candidate_rules[last_exception + 1 :]
+    # New root protections must also cover the Dockerfile-specific context,
+    # including secrets inside otherwise allowed plugin directories.
+    assert set(root_rules) - {"plugins", ""} <= set(exclusions)
+    assert {"baselines", "meta-reme", "output", "reme_studio"} <= set(exclusions)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("archive_error", [False, True])
+async def test_workspace_archiving_keeps_event_loop_responsive(tmp_path, monkeypatch, archive_error):
+    backend = AsyncMock()
+    backend.exec_shell.return_value = SimpleNamespace(exit_code=0, stdout=b"", stderr=b"", ok=lambda: True)
+    case = sandbox_module.DockerReMeSandbox(
+        case_id="archive-thread",
+        container_id="unused",
+        candidate=ImageCandidate("unused"),
+        workspace=AsyncMock(),
+        config="default",
+        plugins=(),
+        command_timeout=5,
+        environment_names=[],
+    )
+    case.backend = backend
+    source = tmp_path / "memory"
+    source.mkdir()
+    (source / "note.md").write_text("saved memory", encoding="utf-8")
+    loop = asyncio.get_running_loop()
+    started = asyncio.Event()
+    release = threading.Event()
+    event_loop_thread = threading.get_ident()
+
+    def blocking_archive(path):
+        assert threading.get_ident() != event_loop_thread
+        loop.call_soon_threadsafe(started.set)
+        if not release.wait(timeout=5):
+            raise TimeoutError("test did not release archive worker")
+        if archive_error:
+            raise ValueError("invalid workspace archive")
+        return _archive_workspace(path)
+
+    monkeypatch.setattr(sandbox_module, "_archive_workspace", blocking_archive)
+    upload = asyncio.create_task(case.upload_workspace(source))
+    try:
+        await asyncio.wait_for(started.wait(), timeout=2)
+        await asyncio.sleep(0)
+        assert not upload.done()
+        # No workspace clearing or upload may happen before validation finishes.
+        backend.exec_shell.assert_not_awaited()
+        backend.write_file.assert_not_awaited()
+        release.set()
+        if archive_error:
+            with pytest.raises(ValueError, match="invalid workspace archive"):
+                await upload
+            backend.exec_shell.assert_not_awaited()
+            backend.write_file.assert_not_awaited()
+        else:
+            await upload
+            backend.write_file.assert_awaited_once()
+            archive_path, payload = backend.write_file.await_args.args
+            assert archive_path == sandbox_module.WORKSPACE_ARCHIVE
+            with tarfile.open(fileobj=io.BytesIO(payload), mode="r:gz") as archive:
+                assert archive.extractfile("note.md").read() == b"saved memory"
+    finally:
+        release.set()
+        await asyncio.gather(upload, return_exceptions=True)
+        await case.close()