mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-10-13 04:01:35 +00:00
217 lines
9.5 KiB
Python
217 lines
9.5 KiB
Python
"""Path-preserving Bubblewrap mounts and job integration."""
|
|
|
|
import sys
|
|
from types import SimpleNamespace
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import yaml
|
|
|
|
from agentscope.tool import ExecResult
|
|
|
|
from reme.utils.file_sandbox.path_bubble_backend import PathBubbleBackend
|
|
from reme.components.application_context import ApplicationContext
|
|
from reme.steps.common.python_execute import PythonExecuteStep
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def linux_backend_selection(monkeypatch):
|
|
"""Keep existing integration assertions on the Linux backend."""
|
|
monkeypatch.setattr("reme.utils.file_sandbox.sys.platform", "linux")
|
|
|
|
|
|
def _mounts(argv):
|
|
return [
|
|
(argv[index], argv[index + 1], argv[index + 2])
|
|
for index in range(len(argv) - 2)
|
|
if argv[index] in ("--bind", "--ro-bind")
|
|
]
|
|
|
|
|
|
def test_path_bubble_backend_preserves_host_paths_and_mount_order(tmp_path):
|
|
"""Nested writable mounts override a read-only parent at the same path."""
|
|
output = tmp_path / "output"
|
|
output.mkdir()
|
|
backend = PathBubbleBackend(cwd=tmp_path, read_only_paths=[str(tmp_path)], writable_paths=[str(output)])
|
|
|
|
argv = backend._bwrap_argv(["/bin/sh", "-c", "pwd"], str(tmp_path)) # pylint: disable=protected-access
|
|
mounts = _mounts(argv)
|
|
|
|
assert ("--ro-bind", str(tmp_path), str(tmp_path)) in mounts
|
|
assert ("--bind", str(output), str(output)) in mounts
|
|
assert mounts.index(("--ro-bind", str(tmp_path), str(tmp_path))) < mounts.index(
|
|
("--bind", str(output), str(output)),
|
|
)
|
|
assert argv[-3:] == ["/bin/sh", "-c", "pwd"]
|
|
assert argv[argv.index("--chdir") + 1] == str(tmp_path)
|
|
assert "--share-net" not in argv
|
|
|
|
|
|
def test_path_bubble_backend_adds_workspace_read_only(tmp_path):
|
|
"""The working directory is always accessible inside the sandbox."""
|
|
backend = PathBubbleBackend(cwd=tmp_path, read_only_paths=[], writable_paths=[])
|
|
argv = backend._bwrap_argv(["true"], str(tmp_path)) # pylint: disable=protected-access
|
|
assert ("--ro-bind", str(tmp_path), str(tmp_path)) in _mounts(argv)
|
|
|
|
|
|
@pytest.mark.parametrize("limit", [-1, True, 1.5, "4096"])
|
|
def test_path_bubble_backend_rejects_invalid_memory_limit(tmp_path, limit):
|
|
"""Memory limits must be non-negative whole bytes."""
|
|
with pytest.raises(ValueError, match="memory_limit_bytes"):
|
|
PathBubbleBackend(cwd=tmp_path, read_only_paths=[], writable_paths=[], memory_limit_bytes=limit)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_each_execution_gets_separate_memory_cgroup(tmp_path, monkeypatch):
|
|
"""Each process receives its own cgroup before bubblewrap is executed."""
|
|
backend = PathBubbleBackend(cwd=tmp_path, read_only_paths=[], writable_paths=[], memory_limit_bytes=1024)
|
|
groups = []
|
|
subprocess_options = []
|
|
subprocess_commands = []
|
|
|
|
def create_group():
|
|
group = tmp_path / f"group-{len(groups)}"
|
|
group.mkdir()
|
|
groups.append(group)
|
|
return group
|
|
|
|
async def create_process(*argv, **kwargs):
|
|
subprocess_commands.append(argv)
|
|
subprocess_options.append(kwargs)
|
|
return SimpleNamespace(pid=100 + len(subprocess_options))
|
|
|
|
monkeypatch.setattr(sys, "platform", "linux")
|
|
monkeypatch.setattr("reme.utils.file_sandbox.path_bubble_backend.shutil.which", lambda _: "/usr/bin/bwrap")
|
|
monkeypatch.setattr(backend, "_create_cgroup", create_group)
|
|
monkeypatch.setattr("reme.utils.file_sandbox.path_bubble_backend.asyncio.create_subprocess_exec", create_process)
|
|
first = await backend._start_process(["true"], str(tmp_path)) # pylint: disable=protected-access
|
|
second = await backend._start_process(["true"], str(tmp_path)) # pylint: disable=protected-access
|
|
|
|
assert len(groups) == 2 and groups[0] != groups[1]
|
|
assert all("preexec_fn" not in options for options in subprocess_options)
|
|
for argv, group in zip(subprocess_commands, groups):
|
|
assert argv[:4] == (sys.executable, "-I", "-S", "-c")
|
|
assert argv[5:7] == (str(group / "cgroup.procs"), "bwrap")
|
|
backend._release_cgroup(first) # pylint: disable=protected-access
|
|
backend._release_cgroup(second) # pylint: disable=protected-access
|
|
assert not any(group.exists() for group in groups)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_zero_memory_limit_skips_cgroup(tmp_path, monkeypatch):
|
|
"""A zero limit bypasses cgroup creation and child enrollment."""
|
|
backend = PathBubbleBackend(cwd=tmp_path, read_only_paths=[], writable_paths=[], memory_limit_bytes=0)
|
|
options = {}
|
|
|
|
async def create_process(*_argv, **kwargs):
|
|
options.update(kwargs)
|
|
return SimpleNamespace(pid=123)
|
|
|
|
monkeypatch.setattr(sys, "platform", "linux")
|
|
monkeypatch.setattr("reme.utils.file_sandbox.path_bubble_backend.shutil.which", lambda _: "/usr/bin/bwrap")
|
|
monkeypatch.setattr("reme.utils.file_sandbox.path_bubble_backend.asyncio.create_subprocess_exec", create_process)
|
|
await backend._start_process(["true"], str(tmp_path)) # pylint: disable=protected-access
|
|
assert "preexec_fn" not in options
|
|
|
|
|
|
@pytest.mark.parametrize("path", ["relative", "/", "/does-not-exist"])
|
|
def test_path_bubble_backend_rejects_invalid_mounts(tmp_path, path):
|
|
"""Only existing absolute non-root paths can be mounted."""
|
|
with pytest.raises(ValueError):
|
|
PathBubbleBackend(cwd=tmp_path, read_only_paths=[path], writable_paths=[])
|
|
|
|
|
|
def test_path_bubble_backend_rejects_symlink_and_duplicates(tmp_path):
|
|
"""Ambiguous mount sources are rejected before a process starts."""
|
|
alias = tmp_path.parent / "alias"
|
|
alias.symlink_to(tmp_path)
|
|
try:
|
|
with pytest.raises(ValueError):
|
|
PathBubbleBackend(cwd=tmp_path, read_only_paths=[str(alias)], writable_paths=[])
|
|
with pytest.raises(ValueError):
|
|
PathBubbleBackend(cwd=tmp_path, read_only_paths=[str(tmp_path)], writable_paths=[str(tmp_path)])
|
|
finally:
|
|
alias.unlink()
|
|
|
|
|
|
def test_path_bubble_backend_rejects_replaced_mount_source(tmp_path):
|
|
"""A path swapped after validation cannot become a new mount source."""
|
|
source = tmp_path / "source"
|
|
source.mkdir()
|
|
backend = PathBubbleBackend(cwd=tmp_path, read_only_paths=[str(source)], writable_paths=[])
|
|
source.rename(tmp_path / "old-source")
|
|
source.mkdir()
|
|
|
|
with pytest.raises(RuntimeError, match="sandbox path changed"):
|
|
backend._bwrap_argv(["true"], str(tmp_path)) # pylint: disable=protected-access
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_python_execute_uses_path_backend_when_paths_supplied(tmp_path, monkeypatch):
|
|
"""Python execution uses the same backend and timeout units."""
|
|
seen = {}
|
|
|
|
async def fake_exec(self, command, *, cwd=None, timeout=None):
|
|
seen.update(command=command, cwd=cwd, timeout=timeout, backend=self)
|
|
return ExecResult(exit_code=0, stdout=b"ok\n", stderr=b"")
|
|
|
|
monkeypatch.setattr(PathBubbleBackend, "exec_shell", fake_exec)
|
|
step = PythonExecuteStep(app_context=ApplicationContext(workspace_dir=str(tmp_path)))
|
|
response = await step(code="print('ok')", read_only_paths=[], writable_paths=[str(tmp_path)])
|
|
|
|
assert response.success is True
|
|
assert response.answer == "ok\n"
|
|
assert seen["command"] == [sys.executable, "-c", "print('ok')"]
|
|
assert seen["cwd"] == str(tmp_path)
|
|
assert seen["timeout"] == 60
|
|
assert seen["backend"]._memory_limit_bytes == 4 * 1024**3 # pylint: disable=protected-access
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_python_execute_passes_custom_memory_limit(tmp_path, monkeypatch):
|
|
"""Job parameters reach the backend without changing path selection."""
|
|
seen = {}
|
|
|
|
async def fake_exec(self, _command, *, cwd=None, timeout=None): # pylint: disable=unused-argument
|
|
seen["limit"] = self._memory_limit_bytes # pylint: disable=protected-access
|
|
return ExecResult(exit_code=0, stdout=b"", stderr=b"")
|
|
|
|
monkeypatch.setattr(PathBubbleBackend, "exec_shell", fake_exec)
|
|
step = PythonExecuteStep(app_context=ApplicationContext(workspace_dir=str(tmp_path)))
|
|
response = await step(code="pass", read_only_paths=[], memory_limit_bytes=0)
|
|
assert response.success is True
|
|
assert seen["limit"] == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_step_rejects_invalid_mount_parameters(tmp_path):
|
|
"""Malformed path injection cannot silently disable sandboxing."""
|
|
step = PythonExecuteStep(app_context=ApplicationContext(workspace_dir=str(tmp_path)))
|
|
response = await step(code="print('unsafe')", writable_paths="/tmp")
|
|
assert response.success is False
|
|
assert "writable_paths must be a list" in response.answer
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sandbox_start_failure_does_not_run_on_host(tmp_path, monkeypatch):
|
|
"""Python execution reports a sandbox failure instead of running on the host."""
|
|
|
|
async def unavailable(*_args, **_kwargs):
|
|
raise RuntimeError("bwrap unavailable")
|
|
|
|
monkeypatch.setattr(PathBubbleBackend, "_start_process", unavailable)
|
|
app_context = ApplicationContext(workspace_dir=str(tmp_path))
|
|
python_step = PythonExecuteStep(app_context=app_context)
|
|
python_result = await python_step(code="print('host')", read_only_paths=[])
|
|
|
|
assert python_result.success is False
|
|
assert "bwrap unavailable" in python_result.answer
|
|
|
|
|
|
def test_job_hides_injected_path_policy():
|
|
"""Path policy reaches the Step through context without becoming a tool argument."""
|
|
config_path = Path(__file__).parents[2] / "reme" / "config" / "benchmark.yaml"
|
|
parameters = yaml.safe_load(config_path.read_text(encoding="utf-8"))["jobs"]["python_execute"]["parameters"]
|
|
assert "read_only_paths" not in parameters["properties"]
|
|
assert "writable_paths" not in parameters["properties"]
|
|
assert "default" not in parameters["properties"]["memory_limit_bytes"]
|