Merge 28a384afb5 into a6509ff479

deploy/__init__.py

@@ -0,0 +1 @@
+"""Deployment and benchmark infrastructure."""

deploy/benchmark_docker/Dockerfile.base

@@ -0,0 +1,24 @@
+# Build from repository root:
+# docker build -f deploy/benchmark_docker/Dockerfile.base -t reme-benchmark-base:agentscope-2.0.9 .
+FROM python:3.11-slim
+
+ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends build-essential git ripgrep tar \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install dependencies with an expendable package stub. Each candidate is
+# installed later with --no-deps so its code, not a fixed ReMe wheel, runs.
+WORKDIR /tmp/reme-dependencies
+COPY pyproject.toml README.md ./
+RUN mkdir reme \
+    && printf '__version__ = "0.0.0"\n' > reme/__init__.py \
+    && python -m pip install '.[core]' \
+    && python -m pip uninstall -y reme-ai \
+    && python -c 'import agentscope; assert agentscope.__version__ == "2.0.9"' \
+    && rm -rf /tmp/reme-dependencies
+
+WORKDIR /workspace

deploy/benchmark_docker/Dockerfile.candidate

@@ -0,0 +1,12 @@
+# Build from repository root with this Dockerfile-specific ignore file.
+ARG BASE_IMAGE=reme-benchmark-base:agentscope-2.0.9
+FROM ${BASE_IMAGE}
+
+COPY . /opt/reme
+ARG REME_PLUGIN_PATH=""
+RUN python -m pip install --no-deps --no-build-isolation /opt/reme \
+    && if [ -n "${REME_PLUGIN_PATH}" ]; then \
+        python -m pip install --no-deps --no-build-isolation "/opt/reme/${REME_PLUGIN_PATH}"; \
+    fi
+
+WORKDIR /workspace

deploy/benchmark_docker/Dockerfile.candidate.dockerignore

@@ -0,0 +1,52 @@
+# Only candidate-side benchmark plugins are needed in this image.
+# Add another explicit plugin directory here before selecting it with REME_PLUGIN_PATH.
+plugins/*
+!plugins/lme
+!plugins/beam
+
+# Keep exclusions after plugin exceptions so nested secrets stay excluded.
+.git
+**/.DS_Store
+**/.ssh
+**/.codex
+**/.claude
+**/private*
+**/.env
+**/.env.*
+**/.reme
+**/.venv
+**/venv
+**/__pycache__
+**/*.py[cod]
+**/.pytest_cache
+**/.mypy_cache
+**/.ruff_cache
+**/.coverage*
+**/htmlcov
+**/node_modules
+**/dist
+**/dist-static
+**/.generated
+**/.next
+**/.vite
+**/.wrangler
+**/.cache
+**/*.egg-info
+**/build
+**/logs
+**/*.log
+**/*.tmp
+reme_studio/src/reme_studio/static
+benchmark
+cookbook
+docs
+github-pages
+integrations
+skills
+tests
+
+# Candidate build inputs do not need local benchmark results or Studio sources.
+baselines
+meta-reme
+output
+reme_studio

deploy/benchmark_docker/README.md

@@ -0,0 +1,140 @@
+# Benchmark Docker runner
+
+This package runs ReMe benchmark jobs directly inside isolated AgentScope
+Docker workspaces. It does not start the ReMe HTTP service. Use AgentScope
+**2.0.9** on the host and in the image.
+
+From the repository root:
+
+```bash
+pip install -e '.[benchmark-docker]'
+docker build -f deploy/benchmark_docker/Dockerfile.base \
+  -t reme-benchmark-base:agentscope-2.0.9 .
+```
+
+`SourceCandidate` uploads an immutable source snapshot and installs the root
+project and any explicitly selected plugin projects into a container-local
+environment. The base image contains dependencies, but no ReMe candidate.
+`ImageCandidate` instead uses a prebuilt image with ReMe and plugins installed.
+The included `Dockerfile.candidate` is an example of that image mode.
+
+```python
+import asyncio
+from pathlib import Path
+
+from deploy.benchmark_docker import (
+    BenchmarkDockerPool, CasePlan, DockerReMeSandboxFactory,
+    EvaluationQuery, JobRequest, SourceCandidate, SourceSnapshot,
+)
+
+
+async def main():
+    snapshot = SourceSnapshot.from_directory(".")
+    factory = DockerReMeSandboxFactory(
+        SourceCandidate(snapshot, install_projects=(".", "plugins/lme")),
+        config="benchmark.yaml",
+        plugins=("lme",),
+        env={"LLM_API_KEY": "...", "EMBEDDING_API_KEY": "..."},
+    )
+    plans = [
+        CasePlan(
+            case_id="case-1",
+            build_jobs=(JobRequest("auto_memory", {"session_id": "s1", "messages": []}),),
+            queries=(EvaluationQuery("q1", "Question?", "Private answer"),),
+            export_workspace_to=Path("artifacts/case-1-workspace.tar.gz"),
+            export_evaluation_to=Path("artifacts/case-1-evaluation.tar.gz"),
+        ),
+    ]
+    return await BenchmarkDockerPool(factory, concurrency=2).run(plans)
+
+
+asyncio.run(main())
+```
+
+Each worker completes one case's build and queries before taking the next
+case. A case can set `reuse_workspace_from="case-1"` to receive the source
+case's workspace **as it stood immediately after construction**, or provide
+`workspace_source` pointing at a host workspace directory or a saved archive.
+A reused workspace may be queried without additional build jobs.
+`export_workspace_to` persists a portable snapshot beyond the pool run.
+To build memory once and query it in later runs using the same factory:
+
+```python
+memory = Path("artifacts/memory.tar.gz")
+pool = BenchmarkDockerPool(factory, concurrency=2)
+built = (await pool.run([
+    CasePlan("build-memory", build_jobs=(JobRequest("auto_memory", {
+        "session_id": "s1", "messages": [],
+    }),), export_workspace_to=memory),
+]))[0]
+if not built.success:
+    raise RuntimeError(built.error or "memory build failed")
+
+# This may run later in another process with a compatible factory/config.
+answers = await pool.run([
+    CasePlan("query-memory", workspace_source=memory,
+             queries=(EvaluationQuery("q1", "Question?", "Private answer"),)),
+])
+```
+
+Use compatible candidate code, plugins, and workspace path configuration when
+restoring memory. Uploading a snapshot does not change the receiving config.
+`export_workspace()` archives contain workspace contents directly. Case archives
+from `export_memory_construction()`, `export_evaluation()`, and `export()` carry
+a `manifest.json` format marker and restore only `reme_workspace/`, including
+its Git history; build logs and query artifacts are excluded from the upload.
+Evaluation exports contain the workspace after queries; use a construction or
+workspace export before queries to preserve the original built memory.
+Query-only case exports cannot be uploaded as workspaces. Older full case
+archives remain supported. Older partial archives with a `reme_workspace`
+wrapper but no manifest are rejected; to reuse one, extract it and supply its
+`reme_workspace` directory.
+
+`export_evaluation_to` saves the workspace and query artifacts for both build
+and query-only cases; it includes `build_log` when that phase ran. Set
+`export_build_log_to` to retain the construction log separately. On a failed
+build, the pool also returns its text in `CaseResult.build_log`. `command_timeout`
+(default 1800 seconds) bounds the entire shell command, including worker startup,
+jobs, and Application shutdown. On timeout, cancellation, or a shell backend
+failure, the sandbox force-stops the container without candidate or plugin
+cooperation, attempts to cache the build log for up to 5 seconds, and removes the
+container before propagating the error. Docker stop and removal each have a
+30-second timeout; cleanup failures include the container ID in exception notes.
+Only the cached build log remains available after removal. The pool creates a
+replacement when more cases need a worker.
+Custom `workspace_builder` implementations must provide an async `stop()` that
+force-stops execution while retaining files for diagnostics, in addition to `close()`.
+Replacement requires successful cleanup of the old worker. Cleanup failures
+are reported in `CaseResult.error` and make `success` false for cases that used
+that worker; failures for an unused worker apply to the whole run. Cancellation
+waits for owned-container cleanup before propagating. A failed `close()` retains
+unfinished Docker handles so another call can retry removal.
+The build log also records configuration, construction, build job, checkpoint,
+and Application lifecycle failures, including available tracebacks.
+
+For direct control, call `create_case()` or `create_cases()`, then use
+`run_job()`, `run_build()`, `run_queries()`, `export_workspace()`,
+`upload_workspace()`, `reset_case()`, and `close()`. Always close a case in a
+`finally` block or use its async context manager. Export before closing.
+The default `export("analysis.tar.gz")` also saves available construction and
+query logs under `analysis.tar.gz.artifacts/build_log/` and
+`analysis.tar.gz.artifacts/queries/`. These logs stay outside the archive;
+choose a new destination if that artifacts directory already exists.
+
+The candidate receives only query ID, question, answer Job, and answer
+arguments. Golden answers, rubrics, and scoring remain on the host. Workspace
+archives reject links, special files, duplicate paths, and traversal paths.
+Source snapshots exclude VCS state, credentials, benchmark data, and generated
+artifacts; review custom source trees for other secrets before uploading.
+
+`tests/unit/test_benchmark_docker.py` runs without Docker. The credential-free
+container smoke tests build the base image and temporary candidate images when
+explicitly enabled:
+
+```bash
+REME_DOCKER_INTEGRATION=1 pytest tests/integration/test_benchmark_docker.py -v
+```
+
+The host must have AgentScope 2.0.9 and a working Docker daemon. To use an
+already built base image, set `REME_BENCHMARK_TEST_IMAGE` to its tag; the tests
+do not remove a caller-supplied image.

deploy/benchmark_docker/__init__.py

@@ -0,0 +1,20 @@
+"""Docker isolation for benchmark cases; independent of the ReMe service."""
+
+from .candidate import ImageCandidate, SourceCandidate, SourceSnapshot
+from .docker_sandbox import DockerReMeSandbox, DockerReMeSandboxFactory
+from .models import EvaluationQuery, JobRequest, JobResult
+from .pool import BenchmarkDockerPool, CasePlan, CaseResult
+
+__all__ = [
+    "BenchmarkDockerPool",
+    "CasePlan",
+    "CaseResult",
+    "DockerReMeSandbox",
+    "DockerReMeSandboxFactory",
+    "EvaluationQuery",
+    "ImageCandidate",
+    "JobRequest",
+    "JobResult",
+    "SourceCandidate",
+    "SourceSnapshot",
+]

deploy/benchmark_docker/_cleanup.py

@@ -0,0 +1,52 @@
+"""Finish owned-resource cleanup before propagating caller cancellation."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Iterable
+
+
+def format_error(error: BaseException) -> str:
+    """Include cleanup notes when an exception becomes a result string."""
+    return "; ".join([f"{type(error).__name__}: {error}", *getattr(error, "__notes__", ())])
+
+
+async def finish_cleanup(
+    operations: Iterable[Awaitable[None]],
+) -> tuple[list[None | BaseException], asyncio.CancelledError | None]:
+    """Collect all failures, shielding teardown from repeated cancellation.
+
+    The caller must propagate the returned cancellation after reporting errors.
+    Individual cleanup operations may still fail or cancel themselves.
+    """
+
+    async def collect(operation: Awaitable[None]) -> None | BaseException:
+        try:
+            await operation
+        except BaseException as exc:
+            # gather(return_exceptions=True) replaces CancelledError and loses
+            # its notes. Keep the original exception for cleanup diagnostics.
+            return exc
+        return None
+
+    cleanup = asyncio.gather(*(collect(operation) for operation in operations))
+    cancelled = None
+    while not cleanup.done():
+        try:
+            await asyncio.shield(cleanup)
+        except asyncio.CancelledError as exc:
+            cancelled = exc
+    return cleanup.result(), cancelled
+
+
+async def cleanup_on_error(operations: Iterable[tuple[str, Awaitable[None]]], error: BaseException) -> None:
+    """Attach teardown failures to the primary error before propagating cancellation."""
+    operations = list(operations)
+    outcomes, cancelled = await finish_cleanup(operation for _, operation in operations)
+    for (name, _), outcome in zip(operations, outcomes):
+        if isinstance(outcome, BaseException):
+            error.add_note(f"{name} close failed: {format_error(outcome)}")
+    if cancelled is not None and not isinstance(error, asyncio.CancelledError):
+        for note in getattr(error, "__notes__", ()):
+            cancelled.add_note(note)
+        raise cancelled from error

deploy/benchmark_docker/candidate.py

@@ -0,0 +1,186 @@
+"""Candidate-code inputs shared by one or more isolated case sandboxes."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import fnmatch
+import gzip
+import hashlib
+import io
+import os
+from pathlib import Path
+from pathlib import PurePosixPath
+import tarfile
+
+DEFAULT_EXCLUDES = (
+    ".git",
+    ".reme",
+    ".env",
+    ".env.*",
+    ".ssh",
+    ".codex",
+    ".claude",
+    ".venv",
+    "venv",
+    "__pycache__",
+    ".pytest_cache",
+    ".mypy_cache",
+    ".ruff_cache",
+    "datasets",
+    "benchmark",
+    "baselines",
+    "meta-reme",
+    "output",
+    "tests",
+    "cookbook",
+    "docs",
+    "github-pages",
+    "integrations",
+    "skills",
+    "reme_studio",
+    "node_modules",
+    "memory_workspaces",
+    "logs",
+    "artifacts",
+    "build",
+    "dist",
+    "*.egg-info",
+    "*.pyc",
+    "*.pyo",
+    "*.log",
+)
+
+
+def _is_excluded(relative_path: Path, patterns: tuple[str, ...]) -> bool:
+    """Return whether any path component or full POSIX path is excluded."""
+    posix = relative_path.as_posix()
+    return any(
+        fnmatch.fnmatch(posix, pattern) or any(fnmatch.fnmatch(part, pattern) for part in relative_path.parts)
+        for pattern in patterns
+    )
+
+
+def _normalized_info(path: Path, arcname: str) -> tarfile.TarInfo:
+    """Create deterministic tar metadata for a regular source file."""
+    stat = path.stat()
+    info = tarfile.TarInfo(arcname)
+    info.size = stat.st_size
+    info.mode = stat.st_mode & 0o777
+    info.mtime = 0
+    info.uid = 0
+    info.gid = 0
+    info.uname = ""
+    info.gname = ""
+    return info
+
+
+@dataclass(frozen=True, slots=True)
+class SourceSnapshot:
+    """A deterministic source archive reusable across many case sandboxes."""
+
+    archive: bytes
+    sha256: str
+    file_count: int
+
+    @classmethod
+    def from_directory(
+        cls,
+        source_dir: str | Path,
+        *,
+        excludes: tuple[str, ...] = DEFAULT_EXCLUDES,
+    ) -> "SourceSnapshot":
+        """Snapshot a candidate directory without runtime or VCS state.
+
+        Symbolic links are rejected instead of followed. This prevents a
+        candidate archive from silently capturing files outside its root.
+        """
+        root = Path(source_dir).resolve(strict=True)
+        if not root.is_dir():
+            raise ValueError(f"candidate source is not a directory: {root}")
+
+        files: list[tuple[Path, Path]] = []
+        # Component matches exclude every descendant. Full-path matches may
+        # exclude only the directory itself, so they must not prune traversal.
+        directory_patterns = tuple(pattern for pattern in excludes if "/" not in pattern)
+        for current, directories, filenames in os.walk(root, topdown=True, followlinks=False):
+            directory = Path(current)
+            for name in directories[:]:
+                path = directory / name
+                if any(fnmatch.fnmatch(name, pattern) for pattern in directory_patterns):
+                    directories.remove(name)
+                elif path.is_symlink() and not _is_excluded(path.relative_to(root), excludes):
+                    raise ValueError(f"candidate source contains a symbolic link: {path.relative_to(root)}")
+            for name in filenames:
+                path = directory / name
+                relative = path.relative_to(root)
+                if _is_excluded(relative, excludes):
+                    continue
+                if path.is_symlink():
+                    raise ValueError(f"candidate source contains a symbolic link: {relative}")
+                if path.is_file():
+                    files.append((relative, path))
+
+        raw = io.BytesIO()
+        with gzip.GzipFile(fileobj=raw, mode="wb", mtime=0) as compressed:
+            with tarfile.open(fileobj=compressed, mode="w") as archive:
+                for relative, path in sorted(files, key=lambda item: item[0].as_posix()):
+                    info = _normalized_info(path, relative.as_posix())
+                    with path.open("rb") as source:
+                        archive.addfile(info, source)
+
+        payload = raw.getvalue()
+        return cls(
+            archive=payload,
+            sha256=hashlib.sha256(payload).hexdigest(),
+            file_count=len(files),
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class SourceCandidate:
+    """A changing source snapshot containing one or more Python projects.
+
+    ``install_projects`` is a trusted, archive-relative installation plan.  A
+    plugin directory merely being present in the snapshot is insufficient:
+    installing it is what publishes its ``reme.plugins`` entry point.
+    """
+
+    snapshot: SourceSnapshot
+    base_image: str = "reme-benchmark-base:agentscope-2.0.9"
+    install_projects: tuple[str, ...] = (".",)
+
+    def __post_init__(self) -> None:
+        if not self.install_projects:
+            raise ValueError("source candidate requires at least one install project")
+        normalized: list[str] = []
+        for value in self.install_projects:
+            if not isinstance(value, str) or not value:
+                raise TypeError("candidate install projects must be non-empty strings")
+            path = PurePosixPath(value)
+            if path.is_absolute() or ".." in path.parts or "\\" in value:
+                raise ValueError(f"candidate install project must stay inside the archive: {value!r}")
+            name = path.as_posix()
+            if name in normalized:
+                raise ValueError(f"candidate install project is duplicated: {value!r}")
+            normalized.append(name)
+        if normalized[0] != ".":
+            raise ValueError("the candidate root project must be installed before plugin projects")
+        object.__setattr__(self, "install_projects", tuple(normalized))
+
+    @property
+    def candidate_id(self) -> str:
+        """Stable identifier for the exact source snapshot."""
+        return self.snapshot.sha256
+
+
+@dataclass(frozen=True, slots=True)
+class ImageCandidate:
+    """A ReMe candidate already installed in an immutable Docker image."""
+
+    image: str
+    candidate_id: str | None = None
+
+    @property
+    def resolved_candidate_id(self) -> str:
+        """Return the caller-supplied digest/ID or the image reference."""
+        return self.candidate_id or self.image

deploy/benchmark_docker/direct_docker_workspace.py

@@ -0,0 +1,118 @@
+"""Minimal AgentScope 2.0.9 DockerWorkspace for direct ReMe jobs.
+
+The stock workspace builds and starts an MCP gateway. The benchmark harness
+uses only DockerBackend's exec/read/write primitives, so building that gateway
+would add an unrelated network-dependent image layer and process.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+from typing import Any
+
+from aiodocker.exceptions import DockerError
+from agentscope.workspace import DockerWorkspace
+
+DOCKER_CLEANUP_TIMEOUT = 30.0
+
+
+class DirectDockerWorkspace(DockerWorkspace):
+    """Use the supplied image directly and skip the unused MCP gateway."""
+
+    _image_tag: str
+    _backend: Any
+    _container: Any
+    _client: Any
+    is_alive: bool
+
+    def __init__(self, **kwargs: Any) -> None:
+        super().__init__(**kwargs)
+        self._lifecycle_lock = asyncio.Lock()
+        self._stopped = False
+
+    async def _build_or_reuse_image(self) -> None:
+        """Select the caller-provided image instead of deriving a gateway image."""
+        self._image_tag = self.base_image
+
+    async def initialize(self) -> None:
+        """Start one Docker container and expose its filesystem/exec backend."""
+        async with self._lifecycle_lock:
+            if self.is_alive:
+                return
+            if self._container is not None or self._client is not None:
+                raise RuntimeError("Docker workspace cleanup must complete before initialization")
+            try:
+                await self._provision_backend()
+                backend = self.get_backend()
+                result = await backend.exec_shell(["mkdir", "-p", self.workdir], cwd="/")
+                if not result.ok():
+                    raise RuntimeError(
+                        f"failed to create Docker workspace {self.workdir!r}: "
+                        f"{result.stderr.decode('utf-8', errors='replace')}",
+                    )
+            except BaseException as exc:
+                try:
+                    await self._teardown_backend()
+                except Exception as close_exc:
+                    exc.add_note(f"workspace close failed: {type(close_exc).__name__}: {close_exc}")
+                raise
+            self.is_alive = True
+            self._stopped = False
+
+    async def stop(self) -> None:
+        """Kill all container processes without asking the candidate to shut down."""
+        async with self._lifecycle_lock:
+            self.is_alive = False
+            if self._container is None or self._stopped:
+                return
+            async with asyncio.timeout(DOCKER_CLEANUP_TIMEOUT):
+                try:
+                    await self._container.kill(signal="SIGKILL")
+                    await self._container.wait(condition="not-running")
+                except DockerError as exc:
+                    if exc.status != 404:
+                        # Docker returns 409 for an already stopped container.
+                        if exc.status != 409 or (await self._container.show())["State"]["Running"]:
+                            raise
+                self._stopped = True
+
+    async def _teardown_backend(self) -> None:
+        """Remove the container, preserving handles until cleanup is confirmed."""
+        self.is_alive = False
+        if self._container is not None:
+            # Preserve AgentScope's bind-mount ownership repair on native Linux.
+            if (
+                not self._stopped
+                and self.host_workdir is not None
+                and sys.platform == "linux"
+                and self._backend is not None
+            ):
+                try:
+                    await self._backend.exec_shell(
+                        ["chown", "-R", f"{os.getuid()}:{os.getgid()}", self.workdir],
+                        timeout=10.0,
+                    )
+                except Exception:
+                    pass
+            try:
+                # Also stop processes if an earlier explicit stop failed, and
+                # retain handles unless Docker confirms removal.
+                await asyncio.wait_for(self._container.delete(force=True), timeout=DOCKER_CLEANUP_TIMEOUT)
+            except DockerError as exc:
+                if exc.status != 404:
+                    raise
+            self._container = None
+            self._backend = None
+        if self._client is not None:
+            await asyncio.wait_for(self._client.close(), timeout=DOCKER_CLEANUP_TIMEOUT)
+            self._client = None
+        self._backend = None
+
+    async def close(self) -> None:
+        """Report cleanup errors and allow another call to retry unfinished work."""
+        # The inherited close clears backend state in a finally block. This
+        # direct workspace has no gateway and must retain state on failure.
+        async with self._lifecycle_lock:
+            await self._teardown_backend()

deploy/benchmark_docker/models.py

@@ -0,0 +1,73 @@
+"""Small serializable contracts shared by the host and sandbox worker."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True, slots=True)
+class JobResult:
+    """Structured result returned by a direct ReMe job invocation.
+
+    ``token_usage`` contains the input/output/total token delta for each
+    configured agent wrapper during this invocation. Missing provider usage is
+    represented by ``None`` at the metric level. ``job_call_counts`` contains
+    non-zero ReMe job invocation deltas, including nested job tools.
+    """
+
+    success: bool
+    answer: Any = ""
+    metadata: dict[str, Any] = field(default_factory=dict)
+    error: str | None = None
+    token_usage: dict[str, dict[str, int | None]] = field(default_factory=dict)
+    job_call_counts: dict[str, int] = field(default_factory=dict)
+
+    @classmethod
+    def from_dict(cls, value: dict[str, Any]) -> "JobResult":
+        """Build a result from the worker's JSON response."""
+        return cls(
+            success=bool(value.get("success")),
+            answer=value.get("answer", ""),
+            metadata=value.get("metadata") if isinstance(value.get("metadata"), dict) else {},
+            token_usage=value.get("token_usage") if isinstance(value.get("token_usage"), dict) else {},
+            job_call_counts=value.get("job_call_counts") if isinstance(value.get("job_call_counts"), dict) else {},
+            error=value.get("error") if isinstance(value.get("error"), str) else None,
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class ActionRecord:
+    """Audit record for one command executed inside a case sandbox."""
+
+    name: str
+    exit_code: int
+    stdout: str
+    stderr: str
+    started_at: str
+    finished_at: str
+
+
+@dataclass(frozen=True, slots=True)
+class JobRequest:
+    """One job executed as part of a shared build Application lifecycle."""
+
+    job: str
+    arguments: dict[str, Any] = field(default_factory=dict)
+    memory_checkpoint: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class EvaluationQuery:
+    """One answer request plus host-only judging metadata.
+
+    Only ``query_id``, ``question``, ``answer_arguments`` and ``answer_job``
+    cross the candidate sandbox boundary.  The remaining fields are retained
+    on the trusted host for post-answer evaluation.
+    """
+
+    query_id: str
+    question: str
+    golden_answer: Any
+    answer_arguments: dict[str, Any] = field(default_factory=dict)
+    answer_job: str = "agentic_answer"

deploy/benchmark_docker/pool.py

@@ -0,0 +1,277 @@
+"""Bounded Docker workers for independent benchmark cases.
+
+Each case completes construction and its queries before its worker is assigned
+another case. A dependent case may reuse the workspace captured immediately
+after its parent's construction, without sharing mutable container state.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any
+
+from ._cleanup import finish_cleanup, format_error
+from .docker_sandbox import DockerReMeSandbox, DockerReMeSandboxFactory
+from .models import EvaluationQuery, JobRequest
+
+
+@dataclass(frozen=True, slots=True)
+class CasePlan:
+    """One case, optionally seeded by a host workspace or another case."""
+
+    case_id: str
+    build_jobs: tuple[JobRequest, ...] = ()
+    queries: tuple[EvaluationQuery, ...] = ()
+    workspace_source: str | Path | None = None
+    reuse_workspace_from: str | None = None
+    export_workspace_to: str | Path | None = None
+    export_build_log_to: str | Path | None = None
+    export_evaluation_to: str | Path | None = None
+
+
+@dataclass(slots=True)
+class CaseResult:
+    """Structured build and answer results; judging stays on the host."""
+
+    case_id: str
+    build: dict[str, Any] | None = None
+    queries: dict[str, Any] | None = None
+    workspace_snapshot: Path | None = None
+    build_log: str | None = None
+    error: str | None = None
+
+    @property
+    def success(self) -> bool:
+        """Whether all requested phases and owned-worker cleanup succeeded."""
+        return (
+            self.error is None
+            and (self.build is None or self.build.get("success") is True)
+            and (self.queries is None or self.queries.get("success") is True)
+        )
+
+
+class BenchmarkDockerPool:
+    """Reuse at most ``concurrency`` containers across case plans.
+
+    ``run`` owns and closes its containers. It never stages all construction
+    phases ahead of queries: each available worker runs build then queries for
+    one case. Dependencies await a completed parent without holding a worker.
+    """
+
+    def __init__(self, factory: DockerReMeSandboxFactory, *, concurrency: int = 4) -> None:
+        if concurrency < 1:
+            raise ValueError("concurrency must be at least 1")
+        self.factory = factory
+        self.concurrency = concurrency
+
+    async def run(self, plans: list[CasePlan]) -> list[CaseResult]:
+        """Return results in input order, closing every container on exit."""
+        if not plans:
+            return []
+        by_id = {plan.case_id: plan for plan in plans}
+        if len(by_id) != len(plans):
+            raise ValueError("case IDs must be unique")
+        for plan in plans:
+            if plan.workspace_source is not None and plan.reuse_workspace_from is not None:
+                raise ValueError(f"case {plan.case_id!r} has two workspace sources")
+            if not plan.build_jobs and plan.workspace_source is None and plan.reuse_workspace_from is None:
+                raise ValueError(f"case {plan.case_id!r} needs build jobs or a workspace source")
+            if plan.reuse_workspace_from is not None and plan.reuse_workspace_from not in by_id:
+                raise ValueError(f"unknown workspace source: {plan.reuse_workspace_from!r}")
+            if plan.export_build_log_to is not None and not plan.build_jobs:
+                raise ValueError("build log export requires build jobs")
+            if plan.export_evaluation_to is not None and not plan.queries:
+                raise ValueError("evaluation export requires queries")
+        self._check_acyclic(by_id)
+        reused_case_ids = {plan.reuse_workspace_from for plan in plans if plan.reuse_workspace_from is not None}
+
+        workers = await self.factory.create_cases(
+            [f"pool-{index}" for index in range(min(self.concurrency, len(plans)))],
+            concurrency=self.concurrency,
+        )
+        available: asyncio.Queue[DockerReMeSandbox | None] = asyncio.Queue()
+        for worker in workers:
+            available.put_nowait(worker)
+        unclaimed = set(by_id)
+        live_slots = len(workers)
+        next_worker_index = len(workers)
+        exhaustion_error = "no available workers"
+        worker_results: dict[int, list[CaseResult]] = {}
+        results: list[CaseResult] = []
+        run_error: BaseException | None = None
+
+        try:
+            with TemporaryDirectory(prefix="reme-benchmark-pool-") as temporary:
+                snapshot_root = Path(temporary)
+                # Tasks are started together; awaiting a parent never consumes
+                # a container. The graph check above rules out deadlocks.
+                tasks: dict[str, asyncio.Task[CaseResult]] = {}
+
+                async def execute(plan: CasePlan) -> CaseResult:
+                    nonlocal live_slots, next_worker_index, exhaustion_error
+                    result = CaseResult(plan.case_id)
+                    source = plan.workspace_source
+                    if plan.reuse_workspace_from is not None:
+                        parent = await tasks[plan.reuse_workspace_from]
+                        if parent.workspace_snapshot is None:
+                            unclaimed.discard(plan.case_id)
+                            return CaseResult(plan.case_id, error=f"source case {parent.case_id!r} failed")
+                        source = parent.workspace_snapshot
+
+                    worker = await available.get()
+                    unclaimed.discard(plan.case_id)
+                    if worker is None:
+                        return CaseResult(plan.case_id, error=exhaustion_error)
+                    worker_results.setdefault(id(worker), []).append(result)
+
+                    def record_lifecycle_error(phase: str, exc: Exception) -> str:
+                        detail = f"{phase} failed: {format_error(exc)}"
+                        result.error = f"{result.error}; {detail}" if result.error else detail
+                        return detail
+
+                    try:
+                        await worker.reset_case(plan.case_id)
+                        if source is not None:
+                            await worker.upload_workspace(source)
+                        if plan.build_jobs:
+                            try:
+                                result.build = await worker.run_build(list(plan.build_jobs))
+                            except Exception as exc:
+                                result.error = format_error(exc)
+                                await self._retain_failed_build_log(worker, plan, result)
+                                return result
+                            if not result.build.get("success"):
+                                result.error = "build failed"
+                                await self._retain_failed_build_log(worker, plan, result)
+                                return result
+                            if plan.export_build_log_to is not None:
+                                await worker.export_build_log(plan.export_build_log_to)
+                        # Capture before queries, since answering may change the
+                        # workspace. Dependents receive construction state.
+                        if plan.export_workspace_to is not None or plan.case_id in reused_case_ids:
+                            destination = plan.export_workspace_to or snapshot_root / f"{plan.case_id}.tar.gz"
+                            result.workspace_snapshot = await worker.export_workspace(destination)
+                        if plan.queries:
+                            result.queries = await worker.run_queries(list(plan.queries))
+                            if not result.queries.get("success"):
+                                result.error = "query failed"
+                        if plan.export_evaluation_to is not None:
+                            await worker.export_evaluation(plan.export_evaluation_to)
+                        return result
+                    except Exception as exc:  # Keep other cases running and release this worker.
+                        result.error = format_error(exc)
+                        return result
+                    finally:
+                        if worker.reusable:
+                            available.put_nowait(worker)
+                        else:
+                            # Confirm cleanup (or retry a failed removal) before
+                            # allowing another case into this slot.
+                            closed = False
+                            try:
+                                await worker.close()
+                            except Exception as exc:
+                                exhaustion_error = (
+                                    f"no available workers: {record_lifecycle_error('worker close', exc)}"
+                                )
+                            else:
+                                closed = True
+
+                            replaced = False
+                            if closed and unclaimed and not asyncio.current_task().cancelling():
+                                worker_index = next_worker_index
+                                next_worker_index += 1
+                                try:
+                                    replacement = await self.factory.create_case(f"pool-{worker_index}")
+                                except Exception as exc:
+                                    exhaustion_error = (
+                                        f"no available workers: {record_lifecycle_error('replacement', exc)}"
+                                    )
+                                else:
+                                    workers.append(replacement)
+                                    available.put_nowait(replacement)
+                                    replaced = True
+                            if not replaced:
+                                live_slots -= 1
+                                if live_slots == 0:
+                                    # Every future case, including one waiting on a
+                                    # parent, must be able to leave available.get().
+                                    for _ in unclaimed:
+                                        available.put_nowait(None)
+
+                for plan in plans:
+                    tasks[plan.case_id] = asyncio.create_task(execute(plan))
+                try:
+                    results = await asyncio.gather(*(tasks[plan.case_id] for plan in plans))
+                finally:
+                    for task in tasks.values():
+                        if not task.done() and not task.cancelling():
+                            task.cancel()
+                    await asyncio.gather(*tasks.values(), return_exceptions=True)
+                for plan, result in zip(plans, results):
+                    if plan.export_workspace_to is None:
+                        result.workspace_snapshot = None  # The internal temporary archive is about to be deleted.
+                return results
+        except BaseException as exc:
+            run_error = exc
+            raise
+        finally:
+            await self._close_workers(workers, worker_results, results, run_error)
+
+    @staticmethod
+    async def _retain_failed_build_log(worker: DockerReMeSandbox, plan: CasePlan, result: CaseResult) -> None:
+        """Keep diagnostics without replacing the original build failure."""
+        try:
+            result.build_log = await worker.read_build_log()
+        except Exception as exc:
+            result.error += f"; build log unavailable: {type(exc).__name__}: {exc}"
+        if plan.export_build_log_to is not None:
+            try:
+                await worker.export_build_log(plan.export_build_log_to)
+            except Exception as exc:
+                result.error += f"; build log export failed: {type(exc).__name__}: {exc}"
+
+    @staticmethod
+    async def _close_workers(
+        workers: list[DockerReMeSandbox],
+        worker_results: dict[int, list[CaseResult]],
+        results: list[CaseResult],
+        run_error: BaseException | None,
+    ) -> None:
+        """Finish all teardown and attach failures to results or the run error."""
+        outcomes, cleanup_cancel = await finish_cleanup(worker.close() for worker in workers)
+        for worker, outcome in zip(workers, outcomes):
+            if isinstance(outcome, BaseException):
+                detail = f"worker {worker.container_id!r} final close failed: {format_error(outcome)}"
+                error = run_error if run_error is not None else cleanup_cancel
+                if error is not None:
+                    error.add_note(detail)
+                # An unused worker is still owned by this run; report its
+                # failure on all results rather than silently dropping it.
+                for result in worker_results.get(id(worker)) or results:
+                    result.error = f"{result.error}; {detail}" if result.error else detail
+        if cleanup_cancel is not None and run_error is None:
+            raise cleanup_cancel
+
+    @staticmethod
+    def _check_acyclic(plans: dict[str, CasePlan]) -> None:
+        visiting: set[str] = set()
+        visited: set[str] = set()
+
+        def visit(case_id: str) -> None:
+            if case_id in visiting:
+                raise ValueError(f"workspace reuse cycle at {case_id!r}")
+            if case_id in visited:
+                return
+            visiting.add(case_id)
+            parent = plans[case_id].reuse_workspace_from
+            if parent is not None:
+                visit(parent)
+            visiting.remove(case_id)
+            visited.add(case_id)
+
+        for case_id in plans:
+            visit(case_id)

deploy/benchmark_docker/worker.py

@@ -0,0 +1,545 @@
+"""In-container worker that calls ReMe jobs directly, without HTTP.
+
+The host uploads this file into every case sandbox. It deliberately imports
+ReMe only after process startup so source candidates can control exactly which
+ReMe package is loaded.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+import json
+import logging
+import os
+from pathlib import Path
+import subprocess
+import traceback
+from typing import Any
+
+_WORKSPACE_PATH_FIELDS = (
+    "metadata_dir",
+    "session_dir",
+    "mem_session_dir",
+    "resource_dir",
+    "daily_dir",
+    "digest_dir",
+    "dialog_dir",
+)
+_ANALYSIS_EXCLUDE_FIELDS = ("metadata_dir", "resource_dir")
+
+
+def _exception_details(exc: BaseException) -> dict[str, Any]:
+    """Return small, structured exception metadata for host-side classification."""
+
+    details: dict[str, Any] = {
+        "module": type(exc).__module__,
+        "type": type(exc).__name__,
+        "message": str(exc),
+    }
+    response = getattr(exc, "response", None)
+    status_code = getattr(response, "status_code", None)
+    if isinstance(status_code, int):
+        details["status_code"] = status_code
+    error_number = getattr(exc, "errno", None)
+    if isinstance(error_number, int):
+        details["errno"] = error_number
+    return details
+
+
+def _write_runtime_layout(case_root: Path, app_config: Any) -> None:
+    """Persist validated, non-secret paths needed by selective export."""
+    resolved_case_root = case_root.resolve()
+    workspace_root = Path(app_config.workspace_dir).resolve()
+    try:
+        workspace_relative = workspace_root.relative_to(resolved_case_root)
+    except ValueError as exc:
+        raise ValueError(f"sandbox workspace must stay under {resolved_case_root}: {workspace_root}") from exc
+
+    resolved_paths: dict[str, str | None] = {}
+    for field in _WORKSPACE_PATH_FIELDS:
+        configured = (
+            str(Path(app_config.session_dir) / "dialog")
+            if field == "dialog_dir"
+            else str(getattr(app_config, field)).strip()
+        )
+        if not configured:
+            resolved_paths[field] = None
+            continue
+        path = (workspace_root / configured).resolve()
+        try:
+            relative = path.relative_to(resolved_case_root)
+        except ValueError as exc:
+            raise ValueError(f"sandbox config path {field} must stay under {workspace_root}: {path}") from exc
+        if path != workspace_root and workspace_root not in path.parents:
+            raise ValueError(f"sandbox config path {field} must stay under {workspace_root}: {path}")
+        resolved_paths[field] = relative.as_posix()
+
+    analysis_excludes = list(
+        dict.fromkeys(path for field in _ANALYSIS_EXCLUDE_FIELDS if (path := resolved_paths[field]) is not None),
+    )
+    layout = {
+        "workspace_root": workspace_relative.as_posix(),
+        "configured_paths": resolved_paths,
+        "analysis_excludes": analysis_excludes,
+    }
+    (resolved_case_root / "runtime-layout.json").write_text(
+        json.dumps(layout, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+
+
+def _prepare_runtime(request: dict[str, Any]) -> tuple[Path, dict[str, Any]]:
+    """Resolve one request's config after confining process-local output."""
+    case_root = Path(request["case_root"])
+    case_tmp = case_root / "tmp"
+    case_tmp.mkdir(parents=True, exist_ok=True)
+    os.environ["TMPDIR"] = str(case_tmp)
+    os.chdir(case_root)
+
+    # Import after setting TMPDIR so ReMe and its subprocesses cannot leave
+    # case-specific temporary files outside the disposable case root.
+    from reme.config import resolve_app_config
+    from reme.plugin import resolve_plugin_runtime
+    from reme.schema import ApplicationConfig
+
+    plugins = request.get("plugins") or []
+    if not isinstance(plugins, list) or not all(isinstance(name, str) and name for name in plugins):
+        raise TypeError("sandbox request plugins must be a list of non-empty strings")
+    app_config = resolve_app_config(
+        config=request.get("config") or "benchmark.yaml",
+        plugins=plugins,
+        workspace_dir=request["workspace_dir"],
+        log_to_console=False,
+        log_to_file=True,
+    )
+    app_config["environment"] = dict(os.environ)
+    # Apply plugin defaults before validation materializes schema defaults,
+    # otherwise those defaults would override plugin paths and partial jobs.
+    runtime = resolve_plugin_runtime(app_config)
+    resolved_config = ApplicationConfig.model_validate(runtime.config)
+    _write_runtime_layout(case_root, resolved_config)
+    # The build checkpoint reads workspace paths from this mapping. Return the
+    # validated model's dump so optional config fields (including ``daily_dir``)
+    # have the same defaults as the Application instance.
+    return case_root, resolved_config.model_dump()
+
+
+async def _run_job_on_app(app: Any, job: str, arguments: dict[str, Any]) -> dict[str, Any]:
+    """Run one job and return token and ReMe job invocation deltas."""
+    from reme.enumeration import ComponentEnum
+    from reme.utils.evaluation_interface import track_agent_token_usage, track_job_counts
+
+    agent_names = list(app.context.components.get(ComponentEnum.AGENT_WRAPPER, {}))
+    job_names = list(app.context.jobs)
+    token_usage: dict[str, dict[str, int | None]] = {}
+    job_call_counts: dict[str, int] = {}
+    try:
+        with (
+            track_agent_token_usage(agent_names, app.context) as token_usage,
+            track_job_counts(job_names, app.context) as job_call_counts,
+        ):
+            response = await app.run_job(job, **arguments)
+    except Exception as exc:  # Preserve usage accumulated before a job failure.
+        result = {
+            "success": False,
+            "answer": "",
+            "metadata": {},
+            "error": f"{type(exc).__name__}: {exc}",
+            "exception": _exception_details(exc),
+            "traceback": traceback.format_exc(),
+        }
+    else:
+        result = {
+            "success": bool(response.success),
+            "answer": response.answer,
+            "metadata": response.metadata,
+            "error": None if response.success else str(response.answer),
+        }
+    result["token_usage"] = token_usage
+    result["job_call_counts"] = {name: count for name, count in job_call_counts.items() if count}
+    return result
+
+
+def _atomic_write_json(path: Path, value: Any) -> None:
+    """Publish one JSON artifact without exposing a partially written file."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
+    temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, default=str) + "\n", encoding="utf-8")
+    temporary.replace(path)
+
+
+def _start_case_log(path: Path) -> Any:
+    """Attach an exact artifact path to ReMe's configured logger backend."""
+    from reme.utils import get_logger
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    logger = get_logger(log_to_console=False, log_to_file=False, force_init=True)
+    if hasattr(logger, "add"):  # Loguru logger.
+        logger.add(str(path), level="INFO", encoding="utf-8")
+    else:  # Standard library logger, selected by REME_DISABLE_LOGURU.
+        logger.addHandler(logging.FileHandler(path, encoding="utf-8"))
+    return logger
+
+
+def _validate_query_id(query_id: Any) -> str:
+    """Validate an ID used verbatim as a directory name in the artifact."""
+    if not isinstance(query_id, str) or not query_id or len(query_id.encode("utf-8")) > 255:
+        raise ValueError("query_id must be a non-empty string no longer than 255 UTF-8 bytes")
+    if query_id in {".", ".."} or "/" in query_id or "\\" in query_id or "\x00" in query_id:
+        raise ValueError(f"query_id is not a safe directory name: {query_id!r}")
+    if query_id == "summary.json":
+        raise ValueError("query_id conflicts with the queries summary: 'summary.json'")
+    return query_id
+
+
+def _clear_query_state(app: Any) -> None:
+    """Drop request-scoped metadata that must not leak into the next query."""
+    app.context.metadata.pop("tool_contexts", None)
+    app.context.metadata.pop("__search_call_budgets", None)
+
+
+@asynccontextmanager
+async def _application_scope(app: Any, failed_results: list[dict[str, Any]] | None = None) -> AsyncIterator[None]:
+    """Close the app, attaching cleanup diagnostics to already-failed results.
+
+    Propagating exceptions retain cleanup notes; cleanup alone still raises.
+    """
+    error: BaseException | None = None
+    try:
+        yield
+    except BaseException as exc:
+        error = exc
+        raise
+    finally:
+        try:
+            await app.close()
+        except BaseException as exc:
+            if error is None:
+                if isinstance(exc, Exception) and failed_results:
+                    diagnostic = {
+                        "error": f"{type(exc).__name__}: {exc}",
+                        "exception": _exception_details(exc),
+                        "traceback": traceback.format_exc(),
+                    }
+                    for result in failed_results:
+                        result.setdefault("cleanup_errors", []).append(diagnostic)
+                else:
+                    raise
+            else:
+                error.add_note("Application close failed:\n" + "".join(traceback.format_exception(exc, chain=False)))
+
+
+async def _run(request: dict[str, Any]) -> dict[str, Any]:
+    """Construct an Application, run one job, and close it deterministically."""
+    job = request["job"]
+    job_args = request.get("arguments") or {}
+    _, app_config = _prepare_runtime(request)
+    from reme.reme import ReMe
+
+    app = ReMe(**app_config)
+    failed_results: list[dict[str, Any]] = []
+    async with _application_scope(app, failed_results):
+        await app.start()
+        result = await _run_job_on_app(app, job, job_args)
+        if not result["success"]:
+            failed_results.append(result)
+    return result
+
+
+def _append_build_error(build_log: Path, detail: str) -> None:
+    """Keep build diagnostics even when the candidate reconfigures the logger."""
+    with build_log.open("a", encoding="utf-8") as log_file:
+        log_file.write(detail.rstrip() + "\n")
+
+
+async def _run_build(request: dict[str, Any]) -> dict[str, Any]:
+    """Execute all construction jobs in one Application and one build log."""
+    build_log = Path(request["case_root"]) / "build_log" / "build.log"
+    if build_log.exists():
+        raise FileExistsError(f"build artifact already exists: {build_log}")
+    build_log.parent.mkdir(parents=True, exist_ok=True)
+    build_log.touch()
+
+    try:
+        case_root, app_config = _prepare_runtime(request)
+        from reme.reme import ReMe
+        from reme.utils import get_logger
+
+        jobs = request.get("jobs")
+        if not isinstance(jobs, list) or not jobs:
+            raise ValueError("build request requires a non-empty jobs list")
+        app = ReMe(**app_config)
+        logger = None
+        results: list[dict[str, Any]] = []
+        failed_results: list[dict[str, Any]] = []
+        try:
+            async with _application_scope(app, failed_results):
+                # Application.__init__ resets the global logger. Attach afterwards.
+                logger = _start_case_log(build_log)
+                logger.info("[sandbox] build phase started")
+                await app.start()
+                for index, specification in enumerate(jobs):
+                    if not isinstance(specification, dict) or not isinstance(specification.get("job"), str):
+                        raise ValueError(f"invalid build job at index {index}")
+                    arguments = specification.get("arguments") or {}
+                    if not isinstance(arguments, dict):
+                        raise ValueError(f"build job arguments must be an object at index {index}")
+                    result = await _run_job_on_app(app, specification["job"], arguments)
+                    if not result["success"]:
+                        failed_results.append(result)
+                        _append_build_error(
+                            build_log,
+                            f"[sandbox] build job {specification['job']!r} failed: "
+                            f"{result.get('error') or 'job failed'}\n"
+                            f"{result.get('traceback') or ''}",
+                        )
+                    item = {"job": specification["job"], "result": result}
+                    checkpoint = specification.get("memory_checkpoint")
+                    if checkpoint is not None and (not isinstance(checkpoint, str) or not checkpoint.strip()):
+                        raise ValueError(f"invalid memory_checkpoint at index {index}")
+                    if result["success"] and checkpoint is not None:
+                        item["memory_checkpoint"] = _commit_memory_checkpoint(case_root, app_config, checkpoint)
+                    results.append(item)
+            for result in failed_results:
+                for diagnostic in result.get("cleanup_errors", []):
+                    _append_build_error(build_log, diagnostic["traceback"])
+        finally:
+            if logger is not None:
+                logger.info("[sandbox] build phase finished")
+            get_logger(log_to_console=False, log_to_file=False, force_init=True)
+    except BaseException:
+        _append_build_error(build_log, traceback.format_exc())
+        raise
+    return {
+        "success": len(results) == len(jobs) and all(item["result"]["success"] for item in results),
+        "jobs": results,
+    }
+
+
+def _commit_memory_checkpoint(case_root: Path, app_config: dict[str, Any], message: str) -> dict[str, Any]:
+    """Commit one host-declared boundary containing durable file-native memory."""
+
+    workspace_root = Path(app_config["workspace_dir"]).resolve()
+    try:
+        workspace_root.relative_to(case_root.resolve())
+    except ValueError as exc:
+        raise ValueError("memory checkpoint path escapes the runtime workspace") from exc
+    relative_paths: list[str] = []
+    for field in ("daily_dir", "digest_dir", "session_dir"):
+        path = (workspace_root / app_config[field]).resolve()
+        try:
+            relative_path = path.relative_to(workspace_root).as_posix()
+        except ValueError as exc:
+            raise ValueError(f"memory checkpoint {field} escapes the runtime workspace") from exc
+        if not relative_path or relative_path == ".":
+            raise ValueError(f"memory checkpoint {field} must not be the workspace root")
+        if relative_path not in relative_paths:
+            relative_paths.append(relative_path)
+
+    def git(*arguments: str, allowed_exit_codes: tuple[int, ...] = (0,)) -> subprocess.CompletedProcess[str]:
+        result = subprocess.run(
+            ["git", *arguments],
+            cwd=workspace_root,
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        if result.returncode not in allowed_exit_codes:
+            detail = (result.stderr or result.stdout).strip()
+            raise RuntimeError(f"git {' '.join(arguments)} failed: {detail}")
+        return result
+
+    git("add", "-A", "--", *relative_paths)
+    changed_paths = [
+        path
+        for path in relative_paths
+        if git("diff", "--cached", "--quiet", "--", path, allowed_exit_codes=(0, 1)).returncode == 1
+    ]
+    commit = [
+        "-c",
+        "user.name=ReMe Sandbox",
+        "-c",
+        "user.email=reme-sandbox@localhost",
+        "commit",
+        "--quiet",
+        "--allow-empty",
+        "--only",
+        "-m",
+        message,
+    ]
+    if changed_paths:
+        commit.extend(["--", *changed_paths])
+    git(*commit)
+    return {
+        "commit_sha": git("rev-parse", "HEAD").stdout.strip(),
+        "message": message,
+        "paths": relative_paths,
+    }
+
+
+async def _run_checkpoint(request: dict[str, Any]) -> dict[str, Any]:
+    """Commit host-requested memory history using the build checkpoint rules."""
+    message = request.get("message")
+    if not isinstance(message, str) or not message.strip():
+        raise ValueError("memory history commit message must be a non-empty string")
+    case_root, app_config = _prepare_runtime(request)
+    return {"success": True, "memory_checkpoint": _commit_memory_checkpoint(case_root, app_config, message)}
+
+
+async def _run_queries(request: dict[str, Any], *, write_summary: bool = True) -> dict[str, Any]:
+    """Execute answer requests in one candidate Application with isolated logs.
+
+    ``write_summary=False`` is the append-only primitive used by the validation
+    scheduler. It runs exactly one independently leased query without sealing
+    the case-level artifact directory with ``summary.json``.
+    """
+    case_root, app_config = _prepare_runtime(request)
+    from reme.reme import ReMe
+    from reme.utils import get_logger
+
+    queries = request.get("queries")
+    if not isinstance(queries, list) or not queries:
+        raise ValueError("query request requires a non-empty queries list")
+    if not write_summary and len(queries) != 1:
+        raise ValueError("single-query request requires exactly one query")
+    query_ids = [_validate_query_id(query.get("query_id") if isinstance(query, dict) else None) for query in queries]
+    if len(query_ids) != len(set(query_ids)):
+        raise ValueError("query IDs must be unique")
+    queries_root = case_root / "queries"
+    if (queries_root / "summary.json").exists() or any((queries_root / query_id).exists() for query_id in query_ids):
+        raise FileExistsError("query artifacts already exist for this case")
+    queries_root.mkdir(parents=True, exist_ok=True)
+
+    app = ReMe(**app_config)
+    results: list[dict[str, Any]] = []
+    failed_results: list[dict[str, Any]] = []
+    async with _application_scope(app, failed_results):
+        try:
+            await app.start()
+            for query, query_id in zip(queries, query_ids):
+                query_dir = queries_root / query_id
+                query_dir.mkdir(parents=False, exist_ok=False)
+                log_path = query_dir / "answer.log"
+                logger = _start_case_log(log_path)
+                logger.info(f"[sandbox] query started query_id={query_id!r}")
+                answer_result: dict[str, Any] | None = None
+                error: str | None = None
+                exception: dict[str, Any] | None = None
+                try:
+                    question = query.get("question")
+                    if not isinstance(question, str):
+                        raise ValueError(f"query {query_id!r} requires a string question")
+                    answer_arguments = dict(query.get("answer_arguments") or {})
+                    answer_arguments["query"] = question
+                    answer_result = await _run_job_on_app(
+                        app,
+                        str(query.get("answer_job") or "agentic_answer"),
+                        answer_arguments,
+                    )
+                    if not answer_result["success"]:
+                        error = answer_result.get("error") or "answer job failed"
+                        exception = answer_result.get("exception")
+                except Exception as exc:  # Keep later queries runnable and preserve this query's log.
+                    error = f"{type(exc).__name__}: {exc}"
+                    exception = _exception_details(exc)
+                    traceback_text = traceback.format_exc()
+                else:
+                    traceback_text = None
+                finally:
+                    logger.info(f"[sandbox] query finished query_id={query_id!r}")
+                    _clear_query_state(app)
+                    get_logger(log_to_console=False, log_to_file=False, force_init=True)
+
+                result = {
+                    "query_id": query_id,
+                    "question": query.get("question"),
+                    "answer": answer_result.get("answer") if answer_result is not None else None,
+                    "score": None,
+                    "answer_result": answer_result,
+                    "judge_result": None,
+                    "error": error,
+                }
+                if exception is not None:
+                    result["exception"] = exception
+                if traceback_text is not None:
+                    result["traceback"] = traceback_text
+                _atomic_write_json(query_dir / "result.json", result)
+                results.append(result)
+                if error is not None:
+                    failed_results.append(result)
+        finally:
+            get_logger(log_to_console=False, log_to_file=False, force_init=True)
+
+    for result in failed_results:
+        if result.get("cleanup_errors"):
+            _atomic_write_json(queries_root / result["query_id"] / "result.json", result)
+
+    summary = {
+        "schema_version": 1,
+        "case_id": request.get("case_id"),
+        "query_count": len(results),
+        "scored_count": 0,
+        "mean_score": None,
+        "queries": [
+            {
+                "query_id": result["query_id"],
+                "score": result["score"],
+                **({"error": result["error"]} if result["error"] is not None else {}),
+            }
+            for result in results
+        ],
+    }
+    if write_summary:
+        _atomic_write_json(queries_root / "summary.json", summary)
+    return {"success": all(result["error"] is None for result in results), "summary": summary, "queries": results}
+
+
+def main() -> int:
+    """Read one request file and always emit a structured response file."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--request", required=True)
+    parser.add_argument("--response", required=True)
+    args = parser.parse_args()
+
+    request_path = Path(args.request)
+    response_path = Path(args.response)
+    response_path.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        request = json.loads(request_path.read_text(encoding="utf-8"))
+        mode = request.get("mode", "job")
+        if mode == "job":
+            operation = _run(request)
+        elif mode == "build":
+            operation = _run_build(request)
+        elif mode == "queries":
+            operation = _run_queries(request)
+        elif mode == "query":
+            operation = _run_queries(request, write_summary=False)
+        elif mode == "checkpoint":
+            operation = _run_checkpoint(request)
+        else:
+            raise ValueError(f"unknown sandbox worker mode: {mode!r}")
+        result = asyncio.run(operation)
+        exit_code = 0 if result["success"] else 1
+    except Exception as exc:  # The host needs an artifact even for broken candidates.
+        result = {
+            "success": False,
+            "answer": "",
+            "metadata": {},
+            "token_usage": {},
+            "job_call_counts": {},
+            "error": f"{type(exc).__name__}: {exc}",
+            "exception": _exception_details(exc),
+            "traceback": traceback.format_exc(),
+        }
+        exit_code = 1
+
+    response_path.write_text(json.dumps(result, ensure_ascii=False, default=str), encoding="utf-8")
+    return exit_code
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pyproject.toml

@@ -78,6 +78,10 @@ dev = [
     "pytest>=9.1.1",
     "pytest-asyncio>=0.23",
 ]
+benchmark-docker = [
+    "reme-ai[as]",
+    "aiodocker",
+]
 full = [
     "reme-ai[core]",
     "reme-ai[qwen-tokenizer]",
@@ -95,12 +99,13 @@ Repository = "https://github.com/agentscope-ai/ReMe"
 reme = "reme.reme:main"
 
 [tool.setuptools]
-packages = { find = { where = ["."], include = ["reme", "reme.*"], exclude = ["reme_studio*"] } }
+packages = { find = { where = ["."], include = ["reme", "reme.*", "deploy", "deploy.*"], exclude = ["reme_studio*"] } }
 include-package-data = true
 
 [tool.setuptools.package-data]
 "*" = ["py.typed", "**/*.yaml", "**/*.json"]
 "reme.components.tokenizer" = ["stopwords"]
+"deploy.benchmark_docker" = ["Dockerfile.*", "README.md"]
 
 [tool.setuptools.dynamic]
 version = { attr = "reme.__version__" }

tests/integration/test_benchmark_docker.py

@@ -0,0 +1,227 @@
+"""Opt-in, credential-free smoke test against real AgentScope Docker containers.
+
+Run with ``REME_DOCKER_INTEGRATION=1 pytest tests/integration/test_benchmark_docker.py -v``.
+The base image is built from the repository Dockerfile unless
+``REME_BENCHMARK_TEST_IMAGE`` names an existing one. Temporary image tags are
+removed after the test; workspace files stay under pytest's temporary root.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tarfile
+import uuid
+
+import pytest
+
+from deploy.benchmark_docker import (
+    BenchmarkDockerPool,
+    CasePlan,
+    DockerReMeSandboxFactory,
+    EvaluationQuery,
+    ImageCandidate,
+    JobRequest,
+    SourceCandidate,
+    SourceSnapshot,
+)
+
+ROOT = Path(__file__).resolve().parents[2]
+SMOKE_CONFIG = """\
+enable_logo: false
+service:
+  backend: cli
+jobs:
+  smoke_write:
+    backend: base
+    steps:
+      - backend: write_step
+  smoke_read:
+    backend: base
+    steps:
+      - backend: read_step
+"""
+
+
+def _docker(*arguments: str, cwd: Path | None = None) -> None:
+    subprocess.run(["docker", *arguments], cwd=cwd, check=True, timeout=1200)
+
+
+@pytest.fixture(scope="module", name="base_image")
+def _base_image():
+    """Build the repository base image unless the caller supplies one."""
+    if os.environ.get("REME_DOCKER_INTEGRATION") != "1":
+        pytest.skip("set REME_DOCKER_INTEGRATION=1 to run the Docker integration test")
+    supplied = os.environ.get("REME_BENCHMARK_TEST_IMAGE")
+    if supplied:
+        yield supplied
+        return
+
+    tag = f"reme-benchmark-smoke:{uuid.uuid4().hex[:12]}"
+    _docker("build", "-f", "deploy/benchmark_docker/Dockerfile.base", "-t", tag, ".", cwd=ROOT)
+    try:
+        yield tag
+    finally:
+        _docker("image", "rm", tag)
+
+
+def _candidate_source(tmp_path: Path) -> Path:
+    source = tmp_path / "candidate"
+    source.mkdir()
+    shutil.copytree(
+        ROOT / "reme",
+        source / "reme",
+        ignore=shutil.ignore_patterns("__pycache__", "*.pyc", "*.pyo"),
+    )
+    shutil.copy2(ROOT / "pyproject.toml", source / "pyproject.toml")
+    shutil.copy2(ROOT / "README.md", source / "README.md")
+    (source / "smoke.yaml").write_text(SMOKE_CONFIG, encoding="utf-8")
+    return source
+
+
+def _container_exists(name: str) -> bool:
+    result = subprocess.run(
+        ["docker", "container", "inspect", name],
+        capture_output=True,
+        check=False,
+        timeout=30,
+    )
+    return result.returncode == 0
+
+
+def _query(query_id: str) -> EvaluationQuery:
+    return EvaluationQuery(
+        query_id=query_id,
+        question="Read the saved note",
+        golden_answer="docker smoke",
+        answer_job="smoke_read",
+        answer_arguments={"path": "note.md"},
+    )
+
+
+@pytest.mark.asyncio
+async def test_real_docker_timeout_kills_blocked_thread_pool(base_image):
+    """A host deadline must terminate a worker blocked in executor shutdown."""
+    from deploy.benchmark_docker.direct_docker_workspace import DirectDockerWorkspace
+    from deploy.benchmark_docker.docker_sandbox import DockerReMeSandbox, SandboxCommandTimeoutError
+
+    container_id = f"timeout-{uuid.uuid4().hex[:12]}"
+    workspace = DirectDockerWorkspace(workspace_id=container_id, base_image=base_image)
+    case = DockerReMeSandbox(
+        case_id="timeout",
+        container_id=container_id,
+        candidate=ImageCandidate(base_image),
+        workspace=workspace,
+        config="unused",
+        plugins=(),
+        command_timeout=2,
+        environment_names=[],
+    )
+    try:
+        await workspace.initialize()
+        case.backend = workspace.get_backend()
+        # Reproduce the shutdown condition without importing ReMe or a plugin.
+        script = (
+            "import threading; from pathlib import Path; from concurrent.futures import ThreadPoolExecutor; "
+            "pool = ThreadPoolExecutor(1); pool.submit(threading.Event().wait); "
+            "log = Path('/workspace/case/build_log/build.log'); log.parent.mkdir(parents=True); "
+            "log.write_text('waiting for thread pool shutdown'); pool.shutdown(wait=True)"
+        )
+        # pylint: disable=protected-access
+        async with case._operation_lock:
+            with pytest.raises(SandboxCommandTimeoutError):
+                await asyncio.wait_for(case._exec("blocked-close", ["python3", "-c", script]), timeout=75)
+        assert not _container_exists(f"as_ws_{container_id}")
+        assert not case.reusable
+        assert await case.read_build_log() == "waiting for thread pool shutdown"
+    finally:
+        await case.close()
+
+
+@pytest.mark.asyncio
+async def test_real_docker_source_pool_workspace_reuse(base_image, tmp_path):
+    """Run build, query, workspace reuse, and query-only export in a real container."""
+    source = _candidate_source(tmp_path)
+    factory = DockerReMeSandboxFactory(
+        SourceCandidate(SourceSnapshot.from_directory(source), base_image=base_image),
+        config="/workspace/candidate/smoke.yaml",
+    )
+    evaluation = tmp_path / "child-evaluation.tar.gz"
+    plans = [
+        CasePlan(
+            "parent",
+            build_jobs=(JobRequest("smoke_write", {"path": "note.md", "content": "docker smoke"}),),
+            queries=(_query("parent-query"),),
+        ),
+        CasePlan(
+            "child",
+            reuse_workspace_from="parent",
+            queries=(_query("child-query"),),
+            export_evaluation_to=evaluation,
+        ),
+    ]
+
+    results = await asyncio.wait_for(BenchmarkDockerPool(factory, concurrency=1).run(plans), timeout=360)
+
+    assert all(result.success for result in results), results
+    assert "docker smoke" in str(results[1].queries["queries"][0]["answer"])
+    with tarfile.open(evaluation, "r:gz") as archive:
+        names = archive.getnames()
+        assert any(name.endswith("reme_workspace/note.md") for name in names)
+        assert any(name.endswith("queries/child-query/answer.log") for name in names)
+        assert not any(name.endswith("build_log/build.log") for name in names)
+
+
+@pytest.mark.asyncio
+async def test_real_docker_source_plugin_install(base_image, tmp_path):
+    """Upload and install a plugin project in the candidate environment."""
+    source = _candidate_source(tmp_path)
+    shutil.copytree(
+        ROOT / "plugins" / "lme",
+        source / "plugins" / "lme",
+        ignore=shutil.ignore_patterns("__pycache__", "*.pyc", "*.pyo"),
+    )
+    factory = DockerReMeSandboxFactory(
+        SourceCandidate(
+            SourceSnapshot.from_directory(source),
+            base_image=base_image,
+            install_projects=(".", "plugins/lme"),
+        ),
+        plugins=("lme",),
+        config="/workspace/candidate/smoke.yaml",
+    )
+    async with await asyncio.wait_for(factory.create_case("plugin-smoke"), timeout=180) as case:
+        container_name = f"as_ws_{case.container_id}"
+        assert _container_exists(container_name)
+    assert not _container_exists(container_name)
+
+
+@pytest.mark.asyncio
+async def test_real_docker_prebuilt_image(base_image, tmp_path):
+    """Install a fixed candidate into an image and execute its jobs."""
+    source = _candidate_source(tmp_path)
+    dockerfile = source / "Dockerfile.smoke"
+    dockerfile.write_text(
+        f"FROM {base_image}\nCOPY . /opt/reme\nRUN python -m pip install --no-deps --no-build-isolation /opt/reme\n",
+        encoding="utf-8",
+    )
+    image = f"reme-benchmark-candidate-smoke:{uuid.uuid4().hex[:12]}"
+    _docker("build", "-f", dockerfile.name, "-t", image, ".", cwd=source)
+    try:
+        factory = DockerReMeSandboxFactory(ImageCandidate(image), config="/opt/reme/smoke.yaml")
+        async with await factory.create_case("image-smoke") as case:
+            container_name = f"as_ws_{case.container_id}"
+            build = await asyncio.wait_for(
+                case.run_build([JobRequest("smoke_write", {"path": "note.md", "content": "image smoke"})]),
+                timeout=120,
+            )
+            assert build["success"], build
+            query = await asyncio.wait_for(case.run_queries([_query("image-query")]), timeout=120)
+            assert query["success"], query
+            assert "image smoke" in str(query["queries"][0]["answer"])
+        assert not _container_exists(container_name)
+    finally:
+        _docker("image", "rm", image)

tests/unit/test_benchmark_docker_queries.py

@@ -0,0 +1,65 @@
+"""Host-only query answers never participate in candidate serialization."""
+
+# pylint: disable=missing-function-docstring,protected-access
+
+import json
+import threading
+from unittest.mock import AsyncMock
+
+import pytest
+
+from deploy.benchmark_docker import EvaluationQuery, ImageCandidate
+from deploy.benchmark_docker.docker_sandbox import DockerReMeSandbox
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("method", ["run_query", "run_queries"])
+@pytest.mark.parametrize("as_mapping", [False, True])
+async def test_query_upload_ignores_uncopyable_golden_answer(monkeypatch, method, as_mapping):
+    case = DockerReMeSandbox(
+        case_id="query",
+        container_id="unused",
+        candidate=ImageCandidate("unused"),
+        workspace=AsyncMock(),
+        config="default",
+        plugins=(),
+        command_timeout=5,
+        environment_names=[],
+    )
+    arguments = {"limit": 3}
+    golden_answer = threading.Lock()
+    query = EvaluationQuery("q1", "Question?", golden_answer, arguments, "custom_answer")
+    if as_mapping:
+        query = {
+            "query_id": "q1",
+            "question": "Question?",
+            "golden_answer": golden_answer,
+            "answer_arguments": arguments,
+            "answer_job": "custom_answer",
+            "rubric": golden_answer,
+        }
+    result = {"answer": "candidate answer"}
+    worker = AsyncMock(return_value={"queries": [result]})
+    monkeypatch.setattr(case, "_run_worker_request", worker)
+
+    if method == "run_query":
+        assert await case.run_query(query) == result
+    else:
+        assert await case.run_queries([query]) == {"queries": [result]}
+
+    mode, payload = worker.await_args.args
+    assert mode == ("query" if method == "run_query" else "queries")
+    assert payload == {
+        "queries": [
+            {
+                "query_id": "q1",
+                "question": "Question?",
+                "answer_arguments": {"limit": 3},
+                "answer_job": "custom_answer",
+            },
+        ],
+    }
+    assert payload["queries"][0]["answer_arguments"] is not arguments
+    # A host-only lock is neither deep-copied nor included in the JSON request.
+    json.dumps(payload)
+    assert not golden_answer.locked()

tests/unit/test_benchmark_docker_upload.py

@@ -0,0 +1,95 @@
+"""Candidate build exclusions and nonblocking workspace upload without Docker."""
+
+# pylint: disable=missing-function-docstring,protected-access
+
+import asyncio
+import io
+from pathlib import Path
+import tarfile
+import threading
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from deploy.benchmark_docker import ImageCandidate
+from deploy.benchmark_docker import docker_sandbox as sandbox_module
+from deploy.benchmark_docker.docker_sandbox import _archive_workspace
+
+
+def test_candidate_image_preserves_root_exclusions_after_plugin_exceptions():
+    root = Path(__file__).resolve().parents[2]
+    root_rules = (root / ".dockerignore").read_text(encoding="utf-8").splitlines()
+    candidate_rules = (
+        (root / "deploy/benchmark_docker/Dockerfile.candidate.dockerignore").read_text(encoding="utf-8").splitlines()
+    )
+    exceptions = [rule for rule in candidate_rules if rule.startswith("!")]
+    assert set(exceptions) == {"!plugins/lme", "!plugins/beam"}
+    assert "plugins/*" in candidate_rules
+    last_exception = max(candidate_rules.index(rule) for rule in exceptions)
+    exclusions = candidate_rules[last_exception + 1 :]
+    # New root protections must also cover the Dockerfile-specific context,
+    # including secrets inside otherwise allowed plugin directories.
+    assert set(root_rules) - {"plugins", ""} <= set(exclusions)
+    assert {"baselines", "meta-reme", "output", "reme_studio"} <= set(exclusions)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("archive_error", [False, True])
+async def test_workspace_archiving_keeps_event_loop_responsive(tmp_path, monkeypatch, archive_error):
+    backend = AsyncMock()
+    backend.exec_shell.return_value = SimpleNamespace(exit_code=0, stdout=b"", stderr=b"", ok=lambda: True)
+    case = sandbox_module.DockerReMeSandbox(
+        case_id="archive-thread",
+        container_id="unused",
+        candidate=ImageCandidate("unused"),
+        workspace=AsyncMock(),
+        config="default",
+        plugins=(),
+        command_timeout=5,
+        environment_names=[],
+    )
+    case.backend = backend
+    source = tmp_path / "memory"
+    source.mkdir()
+    (source / "note.md").write_text("saved memory", encoding="utf-8")
+    loop = asyncio.get_running_loop()
+    started = asyncio.Event()
+    release = threading.Event()
+    event_loop_thread = threading.get_ident()
+
+    def blocking_archive(path):
+        assert threading.get_ident() != event_loop_thread
+        loop.call_soon_threadsafe(started.set)
+        if not release.wait(timeout=5):
+            raise TimeoutError("test did not release archive worker")
+        if archive_error:
+            raise ValueError("invalid workspace archive")
+        return _archive_workspace(path)
+
+    monkeypatch.setattr(sandbox_module, "_archive_workspace", blocking_archive)
+    upload = asyncio.create_task(case.upload_workspace(source))
+    try:
+        await asyncio.wait_for(started.wait(), timeout=2)
+        await asyncio.sleep(0)
+        assert not upload.done()
+        # No workspace clearing or upload may happen before validation finishes.
+        backend.exec_shell.assert_not_awaited()
+        backend.write_file.assert_not_awaited()
+        release.set()
+        if archive_error:
+            with pytest.raises(ValueError, match="invalid workspace archive"):
+                await upload
+            backend.exec_shell.assert_not_awaited()
+            backend.write_file.assert_not_awaited()
+        else:
+            await upload
+            backend.write_file.assert_awaited_once()
+            archive_path, payload = backend.write_file.await_args.args
+            assert archive_path == sandbox_module.WORKSPACE_ARCHIVE
+            with tarfile.open(fileobj=io.BytesIO(payload), mode="r:gz") as archive:
+                assert archive.extractfile("note.md").read() == b"saved memory"
+    finally:
+        release.set()
+        await asyncio.gather(upload, return_exceptions=True)
+        await case.close()

tests/unit/test_benchmark_docker_workspace.py

@@ -0,0 +1,160 @@
+"""Direct workspace lifecycle without a Docker daemon."""
+
+# pylint: disable=missing-function-docstring,protected-access,wrong-import-position
+
+import asyncio
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+DockerError = pytest.importorskip("aiodocker.exceptions").DockerError
+from deploy.benchmark_docker.direct_docker_workspace import DirectDockerWorkspace  # noqa: E402
+
+
+def make_workspace():
+    workspace = DirectDockerWorkspace(workspace_id="cleanup-test")
+    workspace._container = SimpleNamespace(kill=AsyncMock(), wait=AsyncMock(), delete=AsyncMock())
+    workspace._client = SimpleNamespace(close=AsyncMock())
+    workspace._backend = SimpleNamespace(exec_shell=AsyncMock())
+    workspace.is_alive = True
+    return workspace
+
+
+@pytest.mark.asyncio
+async def test_stop_kills_and_confirms_exit_before_removal():
+    workspace = make_workspace()
+    container = workspace._container
+    await workspace.stop()
+    container.kill.assert_awaited_once_with(signal="SIGKILL")
+    container.wait.assert_awaited_once_with(condition="not-running")
+    container.delete.assert_not_awaited()
+    assert workspace._backend is not None
+    assert not workspace.is_alive
+    await workspace.stop()
+    assert container.kill.await_count == 1
+    await workspace.close()
+    container.delete.assert_awaited_once_with(force=True)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("operation", ["kill", "delete"])
+async def test_docker_cleanup_timeout_retains_handles(monkeypatch, operation):
+    from deploy.benchmark_docker import direct_docker_workspace
+
+    monkeypatch.setattr(direct_docker_workspace, "DOCKER_CLEANUP_TIMEOUT", 0.01)
+    workspace = make_workspace()
+    container = workspace._container
+
+    async def blocked(**_kwargs):
+        await asyncio.Event().wait()
+
+    getattr(container, operation).side_effect = blocked
+    with pytest.raises(TimeoutError):
+        await (workspace.stop() if operation == "kill" else workspace.close())
+    assert workspace._container is container
+    getattr(container, operation).side_effect = None
+    await workspace.close()
+
+
+@pytest.mark.asyncio
+async def test_delete_failure_retains_handles_and_can_be_retried():
+    workspace = make_workspace()
+    container, client, backend = workspace._container, workspace._client, workspace._backend
+    container.delete.side_effect = [RuntimeError("delete failed"), None]
+
+    with pytest.raises(RuntimeError, match="delete failed"):
+        await workspace.close()
+    assert workspace._container is container
+    assert workspace._client is client
+    assert workspace._backend is backend
+    assert not workspace.is_alive
+    client.close.assert_not_awaited()
+    with pytest.raises(RuntimeError, match="cleanup must complete"):
+        await workspace.initialize()
+
+    await workspace.close()
+    await workspace.close()
+    assert container.delete.await_count == 2
+    container.delete.assert_awaited_with(force=True)
+    client.close.assert_awaited_once()
+    assert workspace._container is workspace._client is workspace._backend is None
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("status", [404, 500])
+async def test_only_missing_container_is_a_successful_delete(status):
+    workspace = make_workspace()
+    container, client = workspace._container, workspace._client
+    container.delete.side_effect = DockerError(status, {"message": "delete failed"})
+    if status == 404:
+        await workspace.close()
+        assert workspace._container is None
+        client.close.assert_awaited_once()
+    else:
+        with pytest.raises(DockerError):
+            await workspace.close()
+        assert workspace._container is container
+        client.close.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_client_close_failure_retries_without_deleting_again():
+    workspace = make_workspace()
+    container, client = workspace._container, workspace._client
+    client.close.side_effect = [RuntimeError("client close failed"), None]
+    with pytest.raises(RuntimeError, match="client close failed"):
+        await workspace.close()
+    assert workspace._container is workspace._backend is None
+    assert workspace._client is client
+
+    await workspace.close()
+    container.delete.assert_awaited_once_with(force=True)
+    assert client.close.await_count == 2
+    assert workspace._client is None
+
+
+@pytest.mark.asyncio
+async def test_cancelled_delete_retains_state_for_retry():
+    workspace = make_workspace()
+    container, client = workspace._container, workspace._client
+    started = asyncio.Event()
+
+    async def block_delete(**_kwargs):
+        started.set()
+        await asyncio.Event().wait()
+
+    container.delete.side_effect = block_delete
+    task = asyncio.create_task(workspace.close())
+    await asyncio.wait_for(started.wait(), timeout=2)
+    task.cancel()
+    with pytest.raises(asyncio.CancelledError):
+        await task
+    assert workspace._container is container
+    assert workspace._client is client
+
+    container.delete.side_effect = None
+    await workspace.close()
+    assert workspace._container is workspace._client is None
+
+
+@pytest.mark.asyncio
+async def test_initialization_failure_keeps_primary_error_and_retries_cleanup(monkeypatch):
+    workspace = make_workspace()
+    container = workspace._container
+    container.delete.side_effect = [RuntimeError("delete failed"), None]
+    client, backend = workspace._client, workspace._backend
+    workspace._container = workspace._client = workspace._backend = None
+    workspace.is_alive = False
+
+    async def provision():
+        workspace._container, workspace._client, workspace._backend = container, client, backend
+
+    monkeypatch.setattr(workspace, "_provision_backend", provision)
+    backend.exec_shell.return_value = SimpleNamespace(ok=lambda: False, stderr=b"mkdir failed")
+    with pytest.raises(RuntimeError, match="mkdir failed") as failure:
+        await workspace.initialize()
+    assert any("delete failed" in note for note in failure.value.__notes__)
+    assert workspace._container is container
+    await workspace.close()
+    assert workspace._container is workspace._client is workspace._backend is None

tests/unit/test_package_versions.py

@@ -62,7 +62,12 @@ def test_studio_packages_have_independent_identity() -> None:
     assert auto_fin_config["project"]["version"] == "0.1.3"
     assert daily_paper_config["project"]["version"] == "0.1.3"
     assert dingtalk_config["project"]["version"] == "0.1.0"
-    assert main_config["tool"]["setuptools"]["packages"]["find"]["include"] == ["reme", "reme.*"]
+    assert main_config["tool"]["setuptools"]["packages"]["find"]["include"] == [
+        "reme",
+        "reme.*",
+        "deploy",
+        "deploy.*",
+    ]
     assert "reme_studio*" in main_config["tool"]["setuptools"]["packages"]["find"]["exclude"]