From 152fb8ac4fbdccb60fc26ed412d8803096d8c663 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=85=B1=E7=89=9B=E8=82=89?= Date: Tue, 27 Oct 2026 15:40:40 +0800 Subject: [PATCH] fix: align Home dependencies and preserve cleanup cancellation Require ReMe 0.4.2.1 and verify isolated Home wheel installation in package CI. Share cancellation-safe cleanup with thread calls, retain AnyIO cancellation semantics, and record Bubblewrap mount targets during argument construction. --- .github/workflows/_build-python-packages.yml | 36 ++++++++++ .github/workflows/ci-packages.yml | 2 + plugins/home/pyproject.toml | 2 +- reme/__init__.py | 2 +- reme/utils/async_utils.py | 13 ++-- .../utils/file_sandbox/path_bubble_backend.py | 11 +-- .../file_sandbox/path_seatbelt_backend.py | 2 +- reme/utils/local_backend.py | 20 +----- tests/unit/test_local_backend.py | 69 ++++++++++++++++++- tests/unit/test_package_versions.py | 12 ++++ .../unit/test_sandbox_startup_cancellation.py | 49 +++++++++++++ 11 files changed, 188 insertions(+), 30 deletions(-) diff --git a/.github/workflows/_build-python-packages.yml b/.github/workflows/_build-python-packages.yml index 50feee39..b4bfcf85 100644 --- a/.github/workflows/_build-python-packages.yml +++ b/.github/workflows/_build-python-packages.yml @@ -67,6 +67,42 @@ jobs: cd "${RUNNER_TEMP}" "${RUNNER_TEMP}/reme-package-smoke/bin/python" -c "import reme" + - name: Build and verify Home plugin distribution + run: | + python -m build --outdir dist/home plugins/home + python -m twine check dist/home/* + REME_WHEEL="$(pwd)/$(ls dist/reme/reme_ai-[0-9]*.whl)" + HOME_WHEEL="$(pwd)/$(ls dist/home/reme_home-[0-9]*.whl)" + python -m venv "${RUNNER_TEMP}/reme-home-package-smoke" + "${RUNNER_TEMP}/reme-home-package-smoke/bin/python" -m pip install "${REME_WHEEL}[as]" "${HOME_WHEEL}" + "${RUNNER_TEMP}/reme-home-package-smoke/bin/python" -m pip check + cd "${RUNNER_TEMP}" + "${RUNNER_TEMP}/reme-home-package-smoke/bin/python" -I - <<'PY' + from pathlib import Path + import sys + import reme + import reme_home + from reme.enumeration import ComponentEnum + from reme.plugin import resolve_plugin_runtime + from reme_home import AutoBashStep + + for module in (reme, reme_home): + assert Path(module.__file__).is_relative_to(sys.prefix), module.__file__ + runtime = resolve_plugin_runtime({"plugins": ["home"]}) + assert runtime.registry.get(ComponentEnum.STEP, "auto_bash_step") is AutoBashStep + assert runtime.config["jobs"]["auto_bash"]["steps"] == [{"backend": "auto_bash_step"}] + PY + + - name: Reject incompatible published ReMe for Home + run: | + HOME_WHEEL="$(pwd)/$(ls dist/home/reme_home-[0-9]*.whl)" + if python -m pip install --dry-run --ignore-installed "${HOME_WHEEL}" "reme-ai[as]==0.4.2.0" > "${RUNNER_TEMP}/home-incompatible.log" 2>&1; then + echo 'Home must not install with the published ReMe 0.4.2.0 wheel.' + exit 1 + fi + cat "${RUNNER_TEMP}/home-incompatible.log" + grep -q 'ResolutionImpossible' "${RUNNER_TEMP}/home-incompatible.log" + - name: Verify released core dependencies if: inputs.upload_artifacts run: | diff --git a/.github/workflows/ci-packages.yml b/.github/workflows/ci-packages.yml index b79aa3ae..b8760848 100644 --- a/.github/workflows/ci-packages.yml +++ b/.github/workflows/ci-packages.yml @@ -10,6 +10,7 @@ on: - 'pyproject.toml' - 'README.md' - 'reme/**' + - 'plugins/home/**' - 'scripts/bump_version.py' - 'tests/unit/test_package_versions.py' - 'LICENSE' @@ -22,6 +23,7 @@ on: - 'pyproject.toml' - 'README.md' - 'reme/**' + - 'plugins/home/**' - 'scripts/bump_version.py' - 'tests/unit/test_package_versions.py' - 'LICENSE' diff --git a/plugins/home/pyproject.toml b/plugins/home/pyproject.toml index 82e54aed..77f8c174 100644 --- a/plugins/home/pyproject.toml +++ b/plugins/home/pyproject.toml @@ -5,7 +5,7 @@ description = "Local command execution jobs for ReMe." readme = "README.md" requires-python = ">=3.11" dependencies = [ - "reme-ai[as]>=0.4.1.14", + "reme-ai[as]>=0.4.2.1", "safecmd>=0.1.16,<0.2", ] diff --git a/reme/__init__.py b/reme/__init__.py index 4e483086..9e0606d4 100644 --- a/reme/__init__.py +++ b/reme/__init__.py @@ -1,6 +1,6 @@ """ReMe CLI package.""" -__version__ = "0.4.2.0" +__version__ = "0.4.2.1" from . import config from . import constants diff --git a/reme/utils/async_utils.py b/reme/utils/async_utils.py index 8f54fbcb..e80cb7ea 100644 --- a/reme/utils/async_utils.py +++ b/reme/utils/async_utils.py @@ -1,14 +1,14 @@ """Small asyncio helpers.""" import asyncio -from collections.abc import Callable +from collections.abc import Awaitable, Callable from contextlib import suppress from typing import Any -async def complete_in_thread(func: Callable[..., Any], /, *args) -> Any: - """Finish a side-effecting thread call, including through repeated cancellation.""" - task = asyncio.create_task(asyncio.to_thread(func, *args)) +async def finish_cleanup(operation: Awaitable): + """Finish cleanup before propagating any cancellation received while waiting.""" + task = asyncio.ensure_future(operation) cancellation: asyncio.CancelledError | None = None while not task.done(): try: @@ -25,3 +25,8 @@ async def complete_in_thread(func: Callable[..., Any], /, *args) -> Any: task.result() raise cancellation return task.result() + + +async def complete_in_thread(func: Callable[..., Any], /, *args) -> Any: + """Finish a side-effecting thread call, including through repeated cancellation.""" + return await finish_cleanup(asyncio.to_thread(func, *args)) diff --git a/reme/utils/file_sandbox/path_bubble_backend.py b/reme/utils/file_sandbox/path_bubble_backend.py index aaeba139..12eca37a 100644 --- a/reme/utils/file_sandbox/path_bubble_backend.py +++ b/reme/utils/file_sandbox/path_bubble_backend.py @@ -10,7 +10,7 @@ from pathlib import Path import sys from .file_sandbox_backend import FileSandboxBackend, contains -from ..local_backend import finish_cleanup +from ..async_utils import finish_cleanup _CGROUP_CLEANUP_TIMEOUT = 0.25 @@ -48,28 +48,31 @@ class PathBubbleBackend(FileSandboxBackend): raise ValueError("command must not be empty") self._check_paths(cwd) args = ["bwrap", "--die-with-parent", "--new-session", "--unshare-all", "--proc", "/proc", "--dev", "/dev"] + visible_paths = [] for path in ("/usr", "/usr/local"): if os.path.isdir(path): args.extend(("--ro-bind", path, path)) + visible_paths.append(path) for path in ("/bin", "/sbin", "/lib", "/lib64"): if os.path.islink(path): args.extend(("--symlink", os.readlink(path), path)) + visible_paths.append(path) elif os.path.exists(path): args.extend(("--ro-bind", path, path)) + visible_paths.append(path) args.extend(("--dir", "/etc")) for path in ("/etc/passwd", "/etc/group", "/etc/nsswitch.conf", "/etc/hosts", "/etc/resolv.conf", "/etc/ssl"): if os.path.exists(path): args.extend(("--ro-bind", path, path)) + visible_paths.append(path) args.extend(("--tmpfs", "/tmp", "--tmpfs", "/run")) mounts = [(path, "--ro-bind") for path in self._read_only_paths] mounts += [(path, "--bind") for path in self._writable_paths] for path, mode in sorted(mounts, key=lambda item: (item[0].count("/"), item[0])): args.extend((mode, path, path)) + visible_paths.append(path) # Existing mounts already preserve their internal links. Recreate only # runtime links missing from the sandbox, without mounting their parents. - visible_paths = [ - args[index + 2] for index in range(len(args) - 2) if args[index] in {"--ro-bind", "--bind", "--symlink"} - ] links, _ = self._runtime_paths() for path in sorted(links): if not any(contains(root, path) for root in visible_paths): diff --git a/reme/utils/file_sandbox/path_seatbelt_backend.py b/reme/utils/file_sandbox/path_seatbelt_backend.py index 8d8d9940..f2b9ee5a 100644 --- a/reme/utils/file_sandbox/path_seatbelt_backend.py +++ b/reme/utils/file_sandbox/path_seatbelt_backend.py @@ -17,11 +17,11 @@ import psutil from agentscope.tool import ExecResult from .file_sandbox_backend import FileSandboxBackend, contains +from ..async_utils import finish_cleanup from ..local_backend import ( LocalProcessBackend, OutputLimitExceeded, capture_output, - finish_cleanup, output_limit_result, ) diff --git a/reme/utils/local_backend.py b/reme/utils/local_backend.py index bc118cde..4feed857 100644 --- a/reme/utils/local_backend.py +++ b/reme/utils/local_backend.py @@ -9,28 +9,12 @@ from collections.abc import Awaitable, Callable import psutil from agentscope.tool import ExecResult, LocalBackend +from .async_utils import finish_cleanup + _CLEANUP_TIMEOUT = 0.25 OUTPUT_LIMIT_BYTES = 1024 * 1024 -async def finish_cleanup(operation: Awaitable): - """Finish cleanup before propagating any cancellation received while waiting.""" - task = asyncio.ensure_future(operation) - cancelled = False - while True: - try: - result = await asyncio.shield(task) - break - except asyncio.CancelledError: - cancelled = True - if task.done(): - result = task.result() - break - if cancelled: - raise asyncio.CancelledError - return result - - class OutputLimitExceeded(RuntimeError): """Identify the output stream that exceeded its byte budget.""" diff --git a/tests/unit/test_local_backend.py b/tests/unit/test_local_backend.py index 2280d4db..d6cb10c9 100644 --- a/tests/unit/test_local_backend.py +++ b/tests/unit/test_local_backend.py @@ -3,10 +3,77 @@ import asyncio import os +import anyio import psutil import pytest -from reme.utils.local_backend import LocalProcessBackend +from reme.utils.local_backend import LocalProcessBackend, finish_cleanup + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cancellations", [0, 1, 3]) +@pytest.mark.parametrize("fail", [False, True]) +async def test_finish_cleanup_waits_and_retains_first_cancellation(cancellations, fail): + """Cleanup finishes through repeated cancellation and retains the caller's reason.""" + started, release, finished = asyncio.Event(), asyncio.Event(), asyncio.Event() + + async def operation(): + started.set() + try: + await release.wait() + if fail: + raise RuntimeError("cleanup failed") + return "complete" + finally: + finished.set() + + task = asyncio.create_task(finish_cleanup(operation())) + try: + await asyncio.wait_for(started.wait(), timeout=1) + for index in range(cancellations): + task.cancel("original cancellation" if index == 0 else "cancel again") + await asyncio.sleep(0) + assert not task.done() + assert not finished.is_set() + finally: + release.set() + await asyncio.gather(task, return_exceptions=True) + + assert finished.is_set() + if cancellations: + with pytest.raises(asyncio.CancelledError, match="original cancellation"): + task.result() + elif fail: + with pytest.raises(RuntimeError, match="cleanup failed"): + task.result() + else: + assert task.result() == "complete" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scope_name", ["fail_after", "move_on_after"]) +@pytest.mark.parametrize("fail", [False, True]) +async def test_finish_cleanup_preserves_anyio_cancel_scope(scope_name, fail): + """Cancellation first received during cleanup still belongs to its AnyIO scope.""" + finished = asyncio.Event() + + async def operation(): + await asyncio.sleep(0.05) + finished.set() + if fail: + raise RuntimeError("cleanup failed after cancellation") + + async def run_scope(): + with getattr(anyio, scope_name)(0.01) as scope: + await finish_cleanup(operation()) + assert scope.cancelled_caught + + if scope_name == "fail_after": + with pytest.raises(TimeoutError): + await run_scope() + else: + await run_scope() + assert finished.is_set() @pytest.mark.asyncio diff --git a/tests/unit/test_package_versions.py b/tests/unit/test_package_versions.py index 780bd352..bb9bb80f 100644 --- a/tests/unit/test_package_versions.py +++ b/tests/unit/test_package_versions.py @@ -27,6 +27,18 @@ bump_version = _load_script("bump_version") package_studio = _load_script("package_studio") +def test_home_requires_first_sandbox_release() -> None: + """Home must reject published ReMe wheels that predate its sandbox imports.""" + config = tomllib.loads((REPOSITORY / "plugins" / "home" / "pyproject.toml").read_text(encoding="utf-8")) + requirement = next(Requirement(value) for value in config["project"]["dependencies"] if value.startswith("reme-ai")) + + assert requirement.extras == {"as"} + assert Version("0.4.1.14") not in requirement.specifier + assert Version("0.4.2.0") not in requirement.specifier + assert Version("0.4.2.1") in requirement.specifier + assert Version(bump_version.read_version()) in requirement.specifier + + def test_studio_packages_have_independent_identity() -> None: """Keep both Studio distributions independent from the ReMe release version.""" main_config = tomllib.loads((REPOSITORY / "pyproject.toml").read_text(encoding="utf-8")) diff --git a/tests/unit/test_sandbox_startup_cancellation.py b/tests/unit/test_sandbox_startup_cancellation.py index fe65ed61..1b50271f 100644 --- a/tests/unit/test_sandbox_startup_cancellation.py +++ b/tests/unit/test_sandbox_startup_cancellation.py @@ -9,6 +9,7 @@ import sys from pathlib import Path from unittest.mock import AsyncMock +import anyio import pytest from reme.utils.file_sandbox.path_bubble_backend import PathBubbleBackend @@ -16,6 +17,54 @@ from reme.utils.file_sandbox.path_seatbelt_backend import LaunchctlError, PathSe from reme.utils.local_backend import LocalProcessBackend +@pytest.mark.asyncio +@pytest.mark.parametrize("scope_name", ["fail_after", "move_on_after"]) +async def test_anyio_deadline_during_successful_seatbelt_bootout(tmp_path, monkeypatch, scope_name): + """A deadline during normal bootout must retain AnyIO's cancellation identity.""" + backend = PathSeatbeltBackend(cwd=tmp_path, read_only_paths=[], writable_paths=[]) + monkeypatch.setattr("reme.utils.file_sandbox.path_seatbelt_backend.sys.platform", "darwin") + monkeypatch.setattr("reme.utils.file_sandbox.path_seatbelt_backend.shutil.which", lambda _: "sandbox-exec") + monkeypatch.setattr(backend, "_has_gui_launchd", AsyncMock(return_value=True)) + monkeypatch.setattr(backend, "_wait_status", AsyncMock(return_value=0)) + booted_out = False + control = None + scope = None + + async def launchctl(*args): + nonlocal booted_out, control + if args[0] == "bootstrap": + control = Path(args[-1]).parent + elif args[0] == "kickstart": + return "12345" + elif args[0] == "bootout": + assert control.exists() + # Expire only once the completed command enters normal cleanup. + scope.deadline = anyio.current_time() + await asyncio.sleep(0.02) + assert control.exists() + booted_out = True + else: + pytest.fail(f"Unexpected launchctl call: {args[0]}") + return "" + + monkeypatch.setattr(backend, "_launchctl", launchctl) + + async def run_scope(): + nonlocal scope + with getattr(anyio, scope_name)(None) as active_scope: + scope = active_scope + await backend.exec_shell(["true"]) + + if scope_name == "fail_after": + with pytest.raises(TimeoutError): + await run_scope() + else: + await run_scope() + assert scope.cancelled_caught + assert booted_out + assert not control.exists() + + @pytest.mark.asyncio async def test_repeated_cancellation_finishes_fifo_resource_release(tmp_path, monkeypatch): """Even setup failure must drain reader finalizers before closing control resources."""