diff --git a/integrations/hermes_agent/__init__.py b/integrations/hermes_agent/__init__.py index 8dd351a7..2438923e 100644 --- a/integrations/hermes_agent/__init__.py +++ b/integrations/hermes_agent/__init__.py @@ -540,12 +540,19 @@ class ReMeMemoryProvider(MemoryProvider): request_timeout=min(backend_config.request_timeout, remaining), ) self._backend = _backend_for(backend_config) - self._backend.start() + if deadline is None: + self._backend.start() + else: + self._backend.start(deadline=deadline) except (ReMeBackendError, TypeError, ValueError, OSError) as exc: failed, self._backend = self._backend, None if failed is not None: try: - failed.close(timeout=self._shutdown_timeout) + cleanup_timeout = self._shutdown_timeout + if deadline is not None: + cleanup_timeout = max(0.0, deadline - time.monotonic()) + if deadline is None or cleanup_timeout > 0: + failed.close(timeout=cleanup_timeout) except ReMeBackendError as close_exc: logger.warning( "Failed to clean up ReMe after startup error: %s", diff --git a/integrations/hermes_agent/backend.py b/integrations/hermes_agent/backend.py index f491bb4a..9922dffc 100644 --- a/integrations/hermes_agent/backend.py +++ b/integrations/hermes_agent/backend.py @@ -14,8 +14,8 @@ class ReMeBackend(Protocol): label: str - def start(self) -> None: - """Start resources owned by the backend.""" + def start(self, *, deadline: float | None = None) -> None: + """Start owned resources before an optional absolute deadline.""" def health(self, *, timeout: float) -> dict[str, Any]: """Return a semantically healthy ReMe response.""" diff --git a/integrations/hermes_agent/embedded_backend.py b/integrations/hermes_agent/embedded_backend.py index 4335f893..8be5e5a2 100644 --- a/integrations/hermes_agent/embedded_backend.py +++ b/integrations/hermes_agent/embedded_backend.py @@ -51,8 +51,11 @@ class EmbeddedReMeBackend: with self._state_lock: return self._state.value - def start(self) -> None: + def start(self, *, deadline: float | None = None) -> None: """Start the loop thread and construct the Application on that loop.""" + start_deadline = time.monotonic() + self.start_timeout + if deadline is not None: + start_deadline = min(start_deadline, deadline) with self._state_lock: if self._state is _State.RUNNING: return @@ -69,28 +72,35 @@ class EmbeddedReMeBackend: ) self._thread.start() - if not self._loop_ready.wait(timeout=self.start_timeout): - self._fail( - TimeoutError("Timed out while starting the embedded ReMe event loop"), - ) - self.close(timeout=self.start_timeout) + if not self._loop_ready.wait(timeout=max(0.0, start_deadline - time.monotonic())): + error = TimeoutError("Timed out while starting the embedded ReMe event loop") + self._fail(error) + self._close_after_failed_start(start_deadline) raise ReMeBackendError( "Timed out while starting the embedded ReMe event loop", - ) + ) from error try: self._submit( self._start_application(), - timeout=self.start_timeout, + timeout=max(0.0, start_deadline - time.monotonic()), allow_starting=True, ) except ReMeBackendError as exc: self._fail(exc) - self.close(timeout=self.start_timeout) + self._close_after_failed_start(start_deadline) raise with self._state_lock: self._state = _State.RUNNING + def _close_after_failed_start(self, deadline: float) -> None: + """Begin cleanup without extending the startup caller's time budget.""" + try: + self.close(timeout=max(0.0, deadline - time.monotonic())) + except ReMeBackendError: + # close() still requests loop shutdown before reporting a timeout. + pass + def _run_loop(self) -> None: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -225,10 +235,10 @@ class EmbeddedReMeBackend: def close(self, *, timeout: float) -> None: """Close the Application, stop its loop, and join its thread.""" - deadline = time.monotonic() + max(0.1, timeout) + deadline = time.monotonic() + max(0.0, timeout) # A context manager cannot express the bounded wait required by shutdown. acquired = self._operation_lock.acquire( # pylint: disable=consider-using-with - timeout=max(0.1, deadline - time.monotonic()), + timeout=max(0.0, deadline - time.monotonic()), ) if not acquired: raise ReMeBackendError( @@ -253,10 +263,15 @@ class EmbeddedReMeBackend: thread = self._thread if loop is not None and loop.is_running() and self._app is not None: - remaining = max(0.1, deadline - time.monotonic()) + remaining = max(0.0, deadline - time.monotonic()) try: - future = asyncio.run_coroutine_threadsafe(self._app.close(), loop) - future.result(timeout=remaining) + if remaining > 0: + future = asyncio.run_coroutine_threadsafe(self._app.close(), loop) + future.result(timeout=remaining) + else: + close_error = TimeoutError( + "Timed out before closing the embedded ReMe Application", + ) except concurrent.futures.TimeoutError: future.cancel() close_error = TimeoutError( diff --git a/integrations/hermes_agent/http_backend.py b/integrations/hermes_agent/http_backend.py index 21243576..9e0b68d6 100644 --- a/integrations/hermes_agent/http_backend.py +++ b/integrations/hermes_agent/http_backend.py @@ -15,8 +15,9 @@ class HttpReMeBackend: self._client = ReMeHttpClient(endpoint, timeout=request_timeout) self.label = self._client.endpoint - def start(self) -> None: + def start(self, *, deadline: float | None = None) -> None: """HTTP service lifecycle is managed outside Hermes.""" + del deadline def _call( self, diff --git a/tests/unit/test_hermes_agent_integration.py b/tests/unit/test_hermes_agent_integration.py index 3fd46fa9..e485451a 100644 --- a/tests/unit/test_hermes_agent_integration.py +++ b/tests/unit/test_hermes_agent_integration.py @@ -7,6 +7,7 @@ from __future__ import annotations import sys import types +import asyncio import importlib import threading import time @@ -464,6 +465,45 @@ def test_recall_timeout_includes_waiting_for_background_write(): provider.shutdown() +def test_recall_timeout_includes_embedded_startup_failure_cleanup(monkeypatch, tmp_path): + import reme + import reme.config + + class SlowApplication: + def __init__(self, **config): + del config + + async def start(self): + await asyncio.sleep(10) + + async def close(self): + await asyncio.sleep(0.4) + + monkeypatch.setattr(reme, "Application", SlowApplication) + monkeypatch.setattr(reme.config, "resolve_app_config", lambda **kwargs: kwargs) + backend = EmbeddedReMeBackend(str(tmp_path), start_timeout=1) + monkeypatch.setattr(PLUGIN_MODULE, "_backend_for", lambda config: backend) + provider = ReMeMemoryProvider() + provider._config = ReMeConfig( + mode="embedded", + workspace_dir=str(tmp_path), + request_timeout=1, + recall_timeout=0.15, + shutdown_timeout=0.6, + ) + provider._recall_timeout = 0.15 + provider._shutdown_timeout = 0.6 + + started = time.monotonic() + assert provider.prefetch("query") == "" + elapsed = time.monotonic() - started + + assert elapsed < 0.25 + assert backend._thread is not None + backend._thread.join(timeout=1) + assert backend._thread.is_alive() is False + + def test_shutdown_discard_keeps_sentinel_for_inflight_writer(): provider = ReMeMemoryProvider() provider._write_queue.put({"session_id": "queued"})