fix(integrations): bound embedded recall startup cleanup

This commit is contained in:
jinli.yl 2026-09-11 00:05:21 +08:00
parent 4eba5cdab1
commit 973e668279
5 changed files with 82 additions and 19 deletions

View file

@ -540,12 +540,19 @@ class ReMeMemoryProvider(MemoryProvider):
request_timeout=min(backend_config.request_timeout, remaining),
)
self._backend = _backend_for(backend_config)
self._backend.start()
if deadline is None:
self._backend.start()
else:
self._backend.start(deadline=deadline)
except (ReMeBackendError, TypeError, ValueError, OSError) as exc:
failed, self._backend = self._backend, None
if failed is not None:
try:
failed.close(timeout=self._shutdown_timeout)
cleanup_timeout = self._shutdown_timeout
if deadline is not None:
cleanup_timeout = max(0.0, deadline - time.monotonic())
if deadline is None or cleanup_timeout > 0:
failed.close(timeout=cleanup_timeout)
except ReMeBackendError as close_exc:
logger.warning(
"Failed to clean up ReMe after startup error: %s",

View file

@ -14,8 +14,8 @@ class ReMeBackend(Protocol):
label: str
def start(self) -> None:
"""Start resources owned by the backend."""
def start(self, *, deadline: float | None = None) -> None:
"""Start owned resources before an optional absolute deadline."""
def health(self, *, timeout: float) -> dict[str, Any]:
"""Return a semantically healthy ReMe response."""

View file

@ -51,8 +51,11 @@ class EmbeddedReMeBackend:
with self._state_lock:
return self._state.value
def start(self) -> None:
def start(self, *, deadline: float | None = None) -> None:
"""Start the loop thread and construct the Application on that loop."""
start_deadline = time.monotonic() + self.start_timeout
if deadline is not None:
start_deadline = min(start_deadline, deadline)
with self._state_lock:
if self._state is _State.RUNNING:
return
@ -69,28 +72,35 @@ class EmbeddedReMeBackend:
)
self._thread.start()
if not self._loop_ready.wait(timeout=self.start_timeout):
self._fail(
TimeoutError("Timed out while starting the embedded ReMe event loop"),
)
self.close(timeout=self.start_timeout)
if not self._loop_ready.wait(timeout=max(0.0, start_deadline - time.monotonic())):
error = TimeoutError("Timed out while starting the embedded ReMe event loop")
self._fail(error)
self._close_after_failed_start(start_deadline)
raise ReMeBackendError(
"Timed out while starting the embedded ReMe event loop",
)
) from error
try:
self._submit(
self._start_application(),
timeout=self.start_timeout,
timeout=max(0.0, start_deadline - time.monotonic()),
allow_starting=True,
)
except ReMeBackendError as exc:
self._fail(exc)
self.close(timeout=self.start_timeout)
self._close_after_failed_start(start_deadline)
raise
with self._state_lock:
self._state = _State.RUNNING
def _close_after_failed_start(self, deadline: float) -> None:
"""Begin cleanup without extending the startup caller's time budget."""
try:
self.close(timeout=max(0.0, deadline - time.monotonic()))
except ReMeBackendError:
# close() still requests loop shutdown before reporting a timeout.
pass
def _run_loop(self) -> None:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
@ -225,10 +235,10 @@ class EmbeddedReMeBackend:
def close(self, *, timeout: float) -> None:
"""Close the Application, stop its loop, and join its thread."""
deadline = time.monotonic() + max(0.1, timeout)
deadline = time.monotonic() + max(0.0, timeout)
# A context manager cannot express the bounded wait required by shutdown.
acquired = self._operation_lock.acquire( # pylint: disable=consider-using-with
timeout=max(0.1, deadline - time.monotonic()),
timeout=max(0.0, deadline - time.monotonic()),
)
if not acquired:
raise ReMeBackendError(
@ -253,10 +263,15 @@ class EmbeddedReMeBackend:
thread = self._thread
if loop is not None and loop.is_running() and self._app is not None:
remaining = max(0.1, deadline - time.monotonic())
remaining = max(0.0, deadline - time.monotonic())
try:
future = asyncio.run_coroutine_threadsafe(self._app.close(), loop)
future.result(timeout=remaining)
if remaining > 0:
future = asyncio.run_coroutine_threadsafe(self._app.close(), loop)
future.result(timeout=remaining)
else:
close_error = TimeoutError(
"Timed out before closing the embedded ReMe Application",
)
except concurrent.futures.TimeoutError:
future.cancel()
close_error = TimeoutError(

View file

@ -15,8 +15,9 @@ class HttpReMeBackend:
self._client = ReMeHttpClient(endpoint, timeout=request_timeout)
self.label = self._client.endpoint
def start(self) -> None:
def start(self, *, deadline: float | None = None) -> None:
"""HTTP service lifecycle is managed outside Hermes."""
del deadline
def _call(
self,

View file

@ -7,6 +7,7 @@ from __future__ import annotations
import sys
import types
import asyncio
import importlib
import threading
import time
@ -464,6 +465,45 @@ def test_recall_timeout_includes_waiting_for_background_write():
provider.shutdown()
def test_recall_timeout_includes_embedded_startup_failure_cleanup(monkeypatch, tmp_path):
import reme
import reme.config
class SlowApplication:
def __init__(self, **config):
del config
async def start(self):
await asyncio.sleep(10)
async def close(self):
await asyncio.sleep(0.4)
monkeypatch.setattr(reme, "Application", SlowApplication)
monkeypatch.setattr(reme.config, "resolve_app_config", lambda **kwargs: kwargs)
backend = EmbeddedReMeBackend(str(tmp_path), start_timeout=1)
monkeypatch.setattr(PLUGIN_MODULE, "_backend_for", lambda config: backend)
provider = ReMeMemoryProvider()
provider._config = ReMeConfig(
mode="embedded",
workspace_dir=str(tmp_path),
request_timeout=1,
recall_timeout=0.15,
shutdown_timeout=0.6,
)
provider._recall_timeout = 0.15
provider._shutdown_timeout = 0.6
started = time.monotonic()
assert provider.prefetch("query") == ""
elapsed = time.monotonic() - started
assert elapsed < 0.25
assert backend._thread is not None
backend._thread.join(timeout=1)
assert backend._thread.is_alive() is False
def test_shutdown_discard_keeps_sentinel_for_inflight_writer():
provider = ReMeMemoryProvider()
provider._write_queue.put({"session_id": "queued"})