fix(benchmark): align Docker runtime with AgentScope 2.0.11

This commit is contained in:
酱牛肉 2026-11-04 15:34:48 +08:00
parent d43b8857ba
commit 0f50c086bc
9 changed files with 24 additions and 24 deletions

View file

@ -1,5 +1,5 @@
# Build from repository root:
# docker build -f deploy/benchmark_docker/Dockerfile.base -t reme-benchmark-base:agentscope-2.0.10 .
# docker build -f deploy/benchmark_docker/Dockerfile.base -t reme-benchmark-base:agentscope-2.0.11 .
FROM python:3.11-slim
ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \
@ -18,7 +18,7 @@ RUN mkdir reme \
&& printf '__version__ = "0.0.0"\n' > reme/__init__.py \
&& python -m pip install '.[core]' \
&& python -m pip uninstall -y reme-ai \
&& python -c 'import agentscope; assert agentscope.__version__ == "2.0.10"' \
&& python -c 'import agentscope; assert agentscope.__version__ == "2.0.11"' \
&& rm -rf /tmp/reme-dependencies
WORKDIR /workspace

View file

@ -1,5 +1,5 @@
# Build from repository root with this Dockerfile-specific ignore file.
ARG BASE_IMAGE=reme-benchmark-base:agentscope-2.0.10
ARG BASE_IMAGE=reme-benchmark-base:agentscope-2.0.11
FROM ${BASE_IMAGE}
COPY . /opt/reme

View file

@ -2,18 +2,18 @@
This package runs ReMe benchmark jobs directly inside isolated AgentScope
Docker workspaces. It does not start the ReMe HTTP service. Use AgentScope
**2.0.10** on the host and in the image.
**2.0.11** on the host and in the image.
Rebuild the base image when upgrading from the previous AgentScope 2.0.9 image;
existing images must also use 2.0.10 to pass candidate validation. New runs use
the `reme-benchmark-base:agentscope-2.0.10` tag by default, including Home evaluations.
Rebuild the base image when upgrading from an older AgentScope image;
existing images must also use 2.0.11 to pass candidate validation. New runs use
the `reme-benchmark-base:agentscope-2.0.11` tag by default, including Home evaluations.
From the repository root:
```bash
pip install -e '.[benchmark-docker]'
docker build -f deploy/benchmark_docker/Dockerfile.base \
-t reme-benchmark-base:agentscope-2.0.10 .
-t reme-benchmark-base:agentscope-2.0.11 .
```
`SourceCandidate` uploads an immutable source snapshot and installs the root
@ -153,6 +153,6 @@ explicitly enabled:
REME_DOCKER_INTEGRATION=1 pytest tests/integration/test_benchmark_docker.py -v
```
The host must have AgentScope 2.0.10 and a working Docker daemon. To use an
The host must have AgentScope 2.0.11 and a working Docker daemon. To use an
already built base image, set `REME_BENCHMARK_TEST_IMAGE` to its tag; the tests
do not remove a caller-supplied image.

View file

@ -146,7 +146,7 @@ class SourceCandidate:
"""
snapshot: SourceSnapshot
base_image: str = "reme-benchmark-base:agentscope-2.0.10"
base_image: str = "reme-benchmark-base:agentscope-2.0.11"
install_projects: tuple[str, ...] = (".",)
def __post_init__(self) -> None:

View file

@ -1,4 +1,4 @@
"""Minimal AgentScope 2.0.10 DockerWorkspace for direct ReMe jobs.
"""Minimal AgentScope 2.0.11 DockerWorkspace for direct ReMe jobs.
The stock workspace builds and starts an MCP gateway. The benchmark harness
uses only DockerBackend's exec/read/write primitives, so building that gateway

View file

@ -239,13 +239,13 @@ def _archive_workspace(source: str | Path) -> bytes:
def _default_workspace_builder(workspace_id: str, image: str, env: dict[str, str]) -> Workspace:
"""Create the AgentScope 2.0.10 Docker backend lazily."""
"""Create the AgentScope 2.0.11 Docker backend lazily."""
import agentscope
from .direct_docker_workspace import DirectDockerWorkspace
if agentscope.__version__ != "2.0.10":
raise RuntimeError(f"benchmark Docker host requires agentscope 2.0.10, got {agentscope.__version__}")
if agentscope.__version__ != "2.0.11":
raise RuntimeError(f"benchmark Docker host requires agentscope 2.0.11, got {agentscope.__version__}")
return DirectDockerWorkspace(
workspace_id=workspace_id,
base_image=image,
@ -495,9 +495,9 @@ class DockerReMeSandbox:
)
result = await self._exec_checked("validate_candidate", [python, "-c", script])
validation = json.loads(_decode(result.stdout))
if validation.get("agentscope") != "2.0.10":
if validation.get("agentscope") != "2.0.11":
raise SandboxCommandError(
f"candidate environment requires agentscope 2.0.10, got {validation.get('agentscope')!r}",
f"candidate environment requires agentscope 2.0.11, got {validation.get('agentscope')!r}",
)
candidate_root = PurePosixPath(CANDIDATE_ROOT)

View file

@ -32,7 +32,7 @@ class EvaluationSpec(SpecModel):
sampling_policy: SamplingPolicy | None = None
concurrency: int = Field(default=4, ge=1)
timeout_seconds: float = Field(default=7200.0, gt=0, allow_inf_nan=False)
base_image: str = Field(default="reme-benchmark-base:agentscope-2.0.10", min_length=1)
base_image: str = Field(default="reme-benchmark-base:agentscope-2.0.11", min_length=1)
fail_fast: bool = False
max_construction_failure_rate: float = Field(default=0.03, ge=0, le=1, allow_inf_nan=False)
max_artifact_bytes: int = Field(default=2_000_000_000, ge=1)

View file

@ -774,7 +774,7 @@ class FakeBackend:
output = json.dumps(
{
"reme": "/workspace/candidate/reme/__init__.py",
"agentscope": "2.0.10",
"agentscope": "2.0.11",
"plugins": {"lme": "/workspace/candidate/plugins/lme/src/reme_lme/__init__.py"},
},
).encode()

View file

@ -14,13 +14,13 @@ from deploy.benchmark_docker.docker_sandbox import (
)
@pytest.mark.parametrize("version", ["2.0.9", "2.0.10"])
@pytest.mark.parametrize("version", ["2.0.9", "2.0.10", "2.0.11"])
def test_host_version_guard(monkeypatch, version):
"""The configured dependency can create a workspace; stale hosts fail early."""
pytest.importorskip("aiodocker")
monkeypatch.setattr("agentscope.__version__", version)
if version == "2.0.9":
with pytest.raises(RuntimeError, match="requires agentscope 2.0.10"):
if version != "2.0.11":
with pytest.raises(RuntimeError, match="requires agentscope 2.0.11"):
_default_workspace_builder("version-test", "test-image", {})
else:
workspace = _default_workspace_builder("version-test", "test-image", {})
@ -28,7 +28,7 @@ def test_host_version_guard(monkeypatch, version):
@pytest.mark.asyncio
@pytest.mark.parametrize("version", ["2.0.9", "2.0.10"])
@pytest.mark.parametrize("version", ["2.0.9", "2.0.10", "2.0.11"])
async def test_candidate_version_guard(monkeypatch, version):
"""Candidate validation rejects stale images even on a supported host."""
sandbox = DockerReMeSandbox(
@ -47,8 +47,8 @@ async def test_candidate_version_guard(monkeypatch, version):
"_exec_checked",
AsyncMock(return_value=SimpleNamespace(stdout=json.dumps(validation).encode())),
)
if version == "2.0.9":
with pytest.raises(SandboxCommandError, match="requires agentscope 2.0.10"):
if version != "2.0.11":
with pytest.raises(SandboxCommandError, match="requires agentscope 2.0.11"):
await sandbox._validate_candidate("python") # pylint: disable=protected-access
else:
await sandbox._validate_candidate("python") # pylint: disable=protected-access