diff --git a/integrations/hermes_agent/embedded_backend.py b/integrations/hermes_agent/embedded_backend.py index 8be5e5a2..5c805147 100644 --- a/integrations/hermes_agent/embedded_backend.py +++ b/integrations/hermes_agent/embedded_backend.py @@ -43,6 +43,7 @@ class EmbeddedReMeBackend: self._loop: asyncio.AbstractEventLoop | None = None self._thread: threading.Thread | None = None self._app: Any = None + self._app_close_future: concurrent.futures.Future[Any] | None = None self._failure: BaseException | None = None @property @@ -148,6 +149,27 @@ class EmbeddedReMeBackend: self._app = None raise + async def _close_application(self, app: Any) -> None: + """Close one Application and release it only after cleanup completes.""" + try: + await app.close() + finally: + with self._state_lock: + if self._app is app: + self._app = None + + def _stop_loop_after_app_close(self, future: concurrent.futures.Future[Any]) -> None: + """Finish deferred shutdown after Application cleanup leaves the foreground budget.""" + try: + future.result() + except BaseException as exc: # pragma: no cover - retained for diagnostics + with self._state_lock: + self._failure = self._failure or exc + with self._state_lock: + loop = self._loop + if loop is not None and loop.is_running(): + loop.call_soon_threadsafe(loop.stop) + def _fail(self, error: BaseException) -> None: with self._state_lock: self._failure = self._failure or error @@ -252,6 +274,7 @@ class EmbeddedReMeBackend: def _close_locked(self, deadline: float) -> None: """Close while holding the operation lock so jobs cannot overlap shutdown.""" close_error: BaseException | None = None + defer_loop_stop = False with self._state_lock: if self._state is _State.CLOSED: return @@ -261,27 +284,30 @@ class EmbeddedReMeBackend: self._state = _State.CLOSING loop = self._loop thread = self._thread + app = self._app - if loop is not None and loop.is_running() and self._app is not None: + if loop is not None and loop.is_running() and app is not None: remaining = max(0.0, deadline - time.monotonic()) - try: - if remaining > 0: - future = asyncio.run_coroutine_threadsafe(self._app.close(), loop) - future.result(timeout=remaining) - else: - close_error = TimeoutError( - "Timed out before closing the embedded ReMe Application", + with self._state_lock: + future = self._app_close_future + if future is None: + future = asyncio.run_coroutine_threadsafe( + self._close_application(app), + loop, ) + self._app_close_future = future + try: + future.result(timeout=remaining) except concurrent.futures.TimeoutError: - future.cancel() + defer_loop_stop = True close_error = TimeoutError( "Timed out while closing the embedded ReMe Application", ) except BaseException as exc: close_error = exc - finally: - self._app = None - if loop is not None and loop.is_running(): + if defer_loop_stop: + future.add_done_callback(self._stop_loop_after_app_close) + if not defer_loop_stop and loop is not None and loop.is_running(): loop.call_soon_threadsafe(loop.stop) if thread is not None and thread is not threading.current_thread(): thread.join(timeout=max(0.0, deadline - time.monotonic())) diff --git a/tests/unit/test_hermes_agent_integration.py b/tests/unit/test_hermes_agent_integration.py index e485451a..6c0d37a5 100644 --- a/tests/unit/test_hermes_agent_integration.py +++ b/tests/unit/test_hermes_agent_integration.py @@ -470,14 +470,21 @@ def test_recall_timeout_includes_embedded_startup_failure_cleanup(monkeypatch, t import reme.config class SlowApplication: + instances = [] + def __init__(self, **config): del config + self.closed = threading.Event() + self.thread_pool = object() + self.__class__.instances.append(self) async def start(self): - await asyncio.sleep(10) + time.sleep(0.25) async def close(self): await asyncio.sleep(0.4) + self.thread_pool = None + self.closed.set() monkeypatch.setattr(reme, "Application", SlowApplication) monkeypatch.setattr(reme.config, "resolve_app_config", lambda **kwargs: kwargs) @@ -502,6 +509,55 @@ def test_recall_timeout_includes_embedded_startup_failure_cleanup(monkeypatch, t assert backend._thread is not None backend._thread.join(timeout=1) assert backend._thread.is_alive() is False + app = SlowApplication.instances[0] + assert app.closed.is_set() + assert app.thread_pool is None + + +def test_embedded_start_timeout_closes_real_application_resources(monkeypatch, tmp_path): + import reme + import reme.config + + from reme.application import Application + from reme.components import BaseComponent + + component_closed = threading.Event() + + class SlowComponent(BaseComponent): + component_type = "slow_test" + + async def _start(self): + time.sleep(0.25) + + async def _close(self): + component_closed.set() + + app = Application( + workspace_dir=str(tmp_path), + service={"backend": "cli"}, + thread_pool_max_workers=1, + enable_logo=False, + log_to_console=False, + log_to_file=False, + ) + component = SlowComponent(name="slow", app_context=app.context) + app.context.components["slow_test"] = {"slow": component} + monkeypatch.setattr(reme, "Application", lambda **config: app) + monkeypatch.setattr(reme.config, "resolve_app_config", lambda **kwargs: kwargs) + backend = EmbeddedReMeBackend(str(tmp_path), start_timeout=1) + + started = time.monotonic() + with pytest.raises(ReMeBackendError, match="timed out"): + backend.start(deadline=started + 0.15) + elapsed = time.monotonic() - started + + assert elapsed < 0.25 + assert backend._thread is not None + backend._thread.join(timeout=1) + assert backend._thread.is_alive() is False + assert component_closed.is_set() + assert component.is_started is False + assert app.context.thread_pool is None def test_shutdown_discard_keeps_sentinel_for_inflight_writer():