fix(integrations): finish embedded application cleanup

This commit is contained in:
jinli.yl 2026-09-11 12:00:01 +08:00
parent 973e668279
commit 7afcb16a04
2 changed files with 95 additions and 13 deletions

View file

@ -43,6 +43,7 @@ class EmbeddedReMeBackend:
self._loop: asyncio.AbstractEventLoop | None = None
self._thread: threading.Thread | None = None
self._app: Any = None
self._app_close_future: concurrent.futures.Future[Any] | None = None
self._failure: BaseException | None = None
@property
@ -148,6 +149,27 @@ class EmbeddedReMeBackend:
self._app = None
raise
async def _close_application(self, app: Any) -> None:
"""Close one Application and release it only after cleanup completes."""
try:
await app.close()
finally:
with self._state_lock:
if self._app is app:
self._app = None
def _stop_loop_after_app_close(self, future: concurrent.futures.Future[Any]) -> None:
"""Finish deferred shutdown after Application cleanup leaves the foreground budget."""
try:
future.result()
except BaseException as exc: # pragma: no cover - retained for diagnostics
with self._state_lock:
self._failure = self._failure or exc
with self._state_lock:
loop = self._loop
if loop is not None and loop.is_running():
loop.call_soon_threadsafe(loop.stop)
def _fail(self, error: BaseException) -> None:
with self._state_lock:
self._failure = self._failure or error
@ -252,6 +274,7 @@ class EmbeddedReMeBackend:
def _close_locked(self, deadline: float) -> None:
"""Close while holding the operation lock so jobs cannot overlap shutdown."""
close_error: BaseException | None = None
defer_loop_stop = False
with self._state_lock:
if self._state is _State.CLOSED:
return
@ -261,27 +284,30 @@ class EmbeddedReMeBackend:
self._state = _State.CLOSING
loop = self._loop
thread = self._thread
app = self._app
if loop is not None and loop.is_running() and self._app is not None:
if loop is not None and loop.is_running() and app is not None:
remaining = max(0.0, deadline - time.monotonic())
try:
if remaining > 0:
future = asyncio.run_coroutine_threadsafe(self._app.close(), loop)
future.result(timeout=remaining)
else:
close_error = TimeoutError(
"Timed out before closing the embedded ReMe Application",
with self._state_lock:
future = self._app_close_future
if future is None:
future = asyncio.run_coroutine_threadsafe(
self._close_application(app),
loop,
)
self._app_close_future = future
try:
future.result(timeout=remaining)
except concurrent.futures.TimeoutError:
future.cancel()
defer_loop_stop = True
close_error = TimeoutError(
"Timed out while closing the embedded ReMe Application",
)
except BaseException as exc:
close_error = exc
finally:
self._app = None
if loop is not None and loop.is_running():
if defer_loop_stop:
future.add_done_callback(self._stop_loop_after_app_close)
if not defer_loop_stop and loop is not None and loop.is_running():
loop.call_soon_threadsafe(loop.stop)
if thread is not None and thread is not threading.current_thread():
thread.join(timeout=max(0.0, deadline - time.monotonic()))

View file

@ -470,14 +470,21 @@ def test_recall_timeout_includes_embedded_startup_failure_cleanup(monkeypatch, t
import reme.config
class SlowApplication:
instances = []
def __init__(self, **config):
del config
self.closed = threading.Event()
self.thread_pool = object()
self.__class__.instances.append(self)
async def start(self):
await asyncio.sleep(10)
time.sleep(0.25)
async def close(self):
await asyncio.sleep(0.4)
self.thread_pool = None
self.closed.set()
monkeypatch.setattr(reme, "Application", SlowApplication)
monkeypatch.setattr(reme.config, "resolve_app_config", lambda **kwargs: kwargs)
@ -502,6 +509,55 @@ def test_recall_timeout_includes_embedded_startup_failure_cleanup(monkeypatch, t
assert backend._thread is not None
backend._thread.join(timeout=1)
assert backend._thread.is_alive() is False
app = SlowApplication.instances[0]
assert app.closed.is_set()
assert app.thread_pool is None
def test_embedded_start_timeout_closes_real_application_resources(monkeypatch, tmp_path):
import reme
import reme.config
from reme.application import Application
from reme.components import BaseComponent
component_closed = threading.Event()
class SlowComponent(BaseComponent):
component_type = "slow_test"
async def _start(self):
time.sleep(0.25)
async def _close(self):
component_closed.set()
app = Application(
workspace_dir=str(tmp_path),
service={"backend": "cli"},
thread_pool_max_workers=1,
enable_logo=False,
log_to_console=False,
log_to_file=False,
)
component = SlowComponent(name="slow", app_context=app.context)
app.context.components["slow_test"] = {"slow": component}
monkeypatch.setattr(reme, "Application", lambda **config: app)
monkeypatch.setattr(reme.config, "resolve_app_config", lambda **kwargs: kwargs)
backend = EmbeddedReMeBackend(str(tmp_path), start_timeout=1)
started = time.monotonic()
with pytest.raises(ReMeBackendError, match="timed out"):
backend.start(deadline=started + 0.15)
elapsed = time.monotonic() - started
assert elapsed < 0.25
assert backend._thread is not None
backend._thread.join(timeout=1)
assert backend._thread.is_alive() is False
assert component_closed.is_set()
assert component.is_started is False
assert app.context.thread_pool is None
def test_shutdown_discard_keeps_sentinel_for_inflight_writer():