mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-16 09:21:50 +00:00
fix(integrations): bound embedded recall startup cleanup
This commit is contained in:
parent
4eba5cdab1
commit
973e668279
5 changed files with 82 additions and 19 deletions
|
|
@ -540,12 +540,19 @@ class ReMeMemoryProvider(MemoryProvider):
|
|||
request_timeout=min(backend_config.request_timeout, remaining),
|
||||
)
|
||||
self._backend = _backend_for(backend_config)
|
||||
self._backend.start()
|
||||
if deadline is None:
|
||||
self._backend.start()
|
||||
else:
|
||||
self._backend.start(deadline=deadline)
|
||||
except (ReMeBackendError, TypeError, ValueError, OSError) as exc:
|
||||
failed, self._backend = self._backend, None
|
||||
if failed is not None:
|
||||
try:
|
||||
failed.close(timeout=self._shutdown_timeout)
|
||||
cleanup_timeout = self._shutdown_timeout
|
||||
if deadline is not None:
|
||||
cleanup_timeout = max(0.0, deadline - time.monotonic())
|
||||
if deadline is None or cleanup_timeout > 0:
|
||||
failed.close(timeout=cleanup_timeout)
|
||||
except ReMeBackendError as close_exc:
|
||||
logger.warning(
|
||||
"Failed to clean up ReMe after startup error: %s",
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ class ReMeBackend(Protocol):
|
|||
|
||||
label: str
|
||||
|
||||
def start(self) -> None:
|
||||
"""Start resources owned by the backend."""
|
||||
def start(self, *, deadline: float | None = None) -> None:
|
||||
"""Start owned resources before an optional absolute deadline."""
|
||||
|
||||
def health(self, *, timeout: float) -> dict[str, Any]:
|
||||
"""Return a semantically healthy ReMe response."""
|
||||
|
|
|
|||
|
|
@ -51,8 +51,11 @@ class EmbeddedReMeBackend:
|
|||
with self._state_lock:
|
||||
return self._state.value
|
||||
|
||||
def start(self) -> None:
|
||||
def start(self, *, deadline: float | None = None) -> None:
|
||||
"""Start the loop thread and construct the Application on that loop."""
|
||||
start_deadline = time.monotonic() + self.start_timeout
|
||||
if deadline is not None:
|
||||
start_deadline = min(start_deadline, deadline)
|
||||
with self._state_lock:
|
||||
if self._state is _State.RUNNING:
|
||||
return
|
||||
|
|
@ -69,28 +72,35 @@ class EmbeddedReMeBackend:
|
|||
)
|
||||
self._thread.start()
|
||||
|
||||
if not self._loop_ready.wait(timeout=self.start_timeout):
|
||||
self._fail(
|
||||
TimeoutError("Timed out while starting the embedded ReMe event loop"),
|
||||
)
|
||||
self.close(timeout=self.start_timeout)
|
||||
if not self._loop_ready.wait(timeout=max(0.0, start_deadline - time.monotonic())):
|
||||
error = TimeoutError("Timed out while starting the embedded ReMe event loop")
|
||||
self._fail(error)
|
||||
self._close_after_failed_start(start_deadline)
|
||||
raise ReMeBackendError(
|
||||
"Timed out while starting the embedded ReMe event loop",
|
||||
)
|
||||
) from error
|
||||
|
||||
try:
|
||||
self._submit(
|
||||
self._start_application(),
|
||||
timeout=self.start_timeout,
|
||||
timeout=max(0.0, start_deadline - time.monotonic()),
|
||||
allow_starting=True,
|
||||
)
|
||||
except ReMeBackendError as exc:
|
||||
self._fail(exc)
|
||||
self.close(timeout=self.start_timeout)
|
||||
self._close_after_failed_start(start_deadline)
|
||||
raise
|
||||
with self._state_lock:
|
||||
self._state = _State.RUNNING
|
||||
|
||||
def _close_after_failed_start(self, deadline: float) -> None:
|
||||
"""Begin cleanup without extending the startup caller's time budget."""
|
||||
try:
|
||||
self.close(timeout=max(0.0, deadline - time.monotonic()))
|
||||
except ReMeBackendError:
|
||||
# close() still requests loop shutdown before reporting a timeout.
|
||||
pass
|
||||
|
||||
def _run_loop(self) -> None:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
|
@ -225,10 +235,10 @@ class EmbeddedReMeBackend:
|
|||
|
||||
def close(self, *, timeout: float) -> None:
|
||||
"""Close the Application, stop its loop, and join its thread."""
|
||||
deadline = time.monotonic() + max(0.1, timeout)
|
||||
deadline = time.monotonic() + max(0.0, timeout)
|
||||
# A context manager cannot express the bounded wait required by shutdown.
|
||||
acquired = self._operation_lock.acquire( # pylint: disable=consider-using-with
|
||||
timeout=max(0.1, deadline - time.monotonic()),
|
||||
timeout=max(0.0, deadline - time.monotonic()),
|
||||
)
|
||||
if not acquired:
|
||||
raise ReMeBackendError(
|
||||
|
|
@ -253,10 +263,15 @@ class EmbeddedReMeBackend:
|
|||
thread = self._thread
|
||||
|
||||
if loop is not None and loop.is_running() and self._app is not None:
|
||||
remaining = max(0.1, deadline - time.monotonic())
|
||||
remaining = max(0.0, deadline - time.monotonic())
|
||||
try:
|
||||
future = asyncio.run_coroutine_threadsafe(self._app.close(), loop)
|
||||
future.result(timeout=remaining)
|
||||
if remaining > 0:
|
||||
future = asyncio.run_coroutine_threadsafe(self._app.close(), loop)
|
||||
future.result(timeout=remaining)
|
||||
else:
|
||||
close_error = TimeoutError(
|
||||
"Timed out before closing the embedded ReMe Application",
|
||||
)
|
||||
except concurrent.futures.TimeoutError:
|
||||
future.cancel()
|
||||
close_error = TimeoutError(
|
||||
|
|
|
|||
|
|
@ -15,8 +15,9 @@ class HttpReMeBackend:
|
|||
self._client = ReMeHttpClient(endpoint, timeout=request_timeout)
|
||||
self.label = self._client.endpoint
|
||||
|
||||
def start(self) -> None:
|
||||
def start(self, *, deadline: float | None = None) -> None:
|
||||
"""HTTP service lifecycle is managed outside Hermes."""
|
||||
del deadline
|
||||
|
||||
def _call(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from __future__ import annotations
|
|||
|
||||
import sys
|
||||
import types
|
||||
import asyncio
|
||||
import importlib
|
||||
import threading
|
||||
import time
|
||||
|
|
@ -464,6 +465,45 @@ def test_recall_timeout_includes_waiting_for_background_write():
|
|||
provider.shutdown()
|
||||
|
||||
|
||||
def test_recall_timeout_includes_embedded_startup_failure_cleanup(monkeypatch, tmp_path):
|
||||
import reme
|
||||
import reme.config
|
||||
|
||||
class SlowApplication:
|
||||
def __init__(self, **config):
|
||||
del config
|
||||
|
||||
async def start(self):
|
||||
await asyncio.sleep(10)
|
||||
|
||||
async def close(self):
|
||||
await asyncio.sleep(0.4)
|
||||
|
||||
monkeypatch.setattr(reme, "Application", SlowApplication)
|
||||
monkeypatch.setattr(reme.config, "resolve_app_config", lambda **kwargs: kwargs)
|
||||
backend = EmbeddedReMeBackend(str(tmp_path), start_timeout=1)
|
||||
monkeypatch.setattr(PLUGIN_MODULE, "_backend_for", lambda config: backend)
|
||||
provider = ReMeMemoryProvider()
|
||||
provider._config = ReMeConfig(
|
||||
mode="embedded",
|
||||
workspace_dir=str(tmp_path),
|
||||
request_timeout=1,
|
||||
recall_timeout=0.15,
|
||||
shutdown_timeout=0.6,
|
||||
)
|
||||
provider._recall_timeout = 0.15
|
||||
provider._shutdown_timeout = 0.6
|
||||
|
||||
started = time.monotonic()
|
||||
assert provider.prefetch("query") == ""
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
assert elapsed < 0.25
|
||||
assert backend._thread is not None
|
||||
backend._thread.join(timeout=1)
|
||||
assert backend._thread.is_alive() is False
|
||||
|
||||
|
||||
def test_shutdown_discard_keeps_sentinel_for_inflight_writer():
|
||||
provider = ReMeMemoryProvider()
|
||||
provider._write_queue.put({"session_id": "queued"})
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue