mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-30 01:52:29 +00:00
a4efc0f776
* fix(bm25_index): 修正BM25索引计算中的文档长度归一化问题 修复了在计算BM25相似度时对文档长度进行不正确归一化的bug，确保所有查询都能得到准确的相关性评分。 * up * up * up * up * up * up * up * up * up * up * up * up * up * up * up * up * up * refactor(steps): Rename and adjust indexing step logic - Rename `scan_changes.py` and `reindex.py` to `clear_and_scan.py` - Update implementation details of `ScanChangesStep` and `ClearAndScanStep` - Modify the scheduling mechanism in `WatchChangesStep` - Adjust step registration and parameter configuration in config files - Update related tests to align with the new interface changes * up * feat(daily): replace daily CRUD operations with slug provisioning approach * refactor(tests): migrate CRUD step tests from HTTP server to direct LocalFileStore * up * up * up * up --------- Co-authored-by: huangsen <huangsen.huang@alibaba-inc.com>
78 lines
2.9 KiB
Python
78 lines
2.9 KiB
Python
"""Long-running awatch loop: convert raw changes into update_index calls."""
|
|
|
|
import asyncio
|
|
|
|
from watchfiles import Change, awatch
|
|
|
|
from ..base_step import BaseStep
|
|
from ...components import R, BaseComponent
|
|
from ...enumeration import ComponentEnum
|
|
|
|
|
|
@R.register("watch_changes_step")
|
|
class WatchChangesStep(BaseStep):
|
|
"""Watch files and forward each batch of raw changes to a downstream step."""
|
|
|
|
def __init__(
|
|
self,
|
|
recursive: bool = True,
|
|
force_polling: bool = True,
|
|
debounce: int = 2000,
|
|
poll_delay_ms: int = 2000,
|
|
dispatch_step: str = "",
|
|
**kwargs,
|
|
):
|
|
super().__init__(**kwargs)
|
|
self.recursive: bool = recursive
|
|
self.force_polling: bool = force_polling
|
|
self.debounce: int = debounce
|
|
self.poll_delay_ms: int = poll_delay_ms
|
|
self.dispatch_step: str = dispatch_step
|
|
|
|
def _filter(self, _change: Change, path: str) -> bool:
|
|
suffixes = (self.context.get("suffix_filters") if self.context else None) or ["md"]
|
|
return not suffixes or any(path.endswith("." + s.strip(".")) for s in suffixes)
|
|
|
|
async def execute(self):
|
|
if self.context is None:
|
|
raise RuntimeError("watch_changes_step requires 'context'")
|
|
if self.context.stop_event is None:
|
|
raise RuntimeError("watch_changes_step requires 'stop_event' on context")
|
|
stop_event: asyncio.Event = self.context.stop_event
|
|
|
|
raw = self.context.get("watch_paths", [])
|
|
paths = [raw] if isinstance(raw, str) else raw
|
|
valid_paths = [self.vault_path / x for x in paths if (self.vault_path / x).exists()]
|
|
if not valid_paths:
|
|
raise RuntimeError(f"No valid watch paths under {self.vault_path}: {paths}")
|
|
|
|
dispatch_step_cls: type[BaseComponent] | None = None
|
|
if self.dispatch_step:
|
|
dispatch_step_cls = R.get(ComponentEnum.STEP, self.dispatch_step)
|
|
if dispatch_step_cls is None:
|
|
raise RuntimeError(f"Unregistered step '{self.dispatch_step}'")
|
|
|
|
self.logger.info(f"Watching: {[str(p) for p in valid_paths]}")
|
|
async for raw_changes in awatch(
|
|
*valid_paths,
|
|
watch_filter=self._filter,
|
|
recursive=self.recursive,
|
|
force_polling=self.force_polling,
|
|
debounce=self.debounce,
|
|
poll_delay_ms=self.poll_delay_ms,
|
|
stop_event=stop_event,
|
|
):
|
|
if stop_event.is_set():
|
|
break
|
|
changes = [
|
|
{"change": c.name, "path": p}
|
|
for c, p in raw_changes
|
|
if c in (Change.added, Change.modified, Change.deleted)
|
|
]
|
|
if changes:
|
|
self.logger.info(f"Detected {len(changes)} change(s)")
|
|
if dispatch_step_cls is not None:
|
|
step = dispatch_step_cls(app_context=self.app_context)
|
|
await step(changes=changes)
|
|
|
|
return self.context.response
|