mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-29 01:41:38 +00:00
041f957a7f
* refactor(components): extract shared component state into mixin - Introduce ComponentMixin class with shared state for components and steps - Move identity, config, and vault path functionality to ComponentMixin - Update BaseComponent to inherit from ComponentMixin - Update BaseStep to inherit from ComponentMixin - Consolidate vault path helper methods in ComponentMixin - Remove duplicate vault path implementations from BaseComponent and BaseStep - Add ComponentMixin to components module exports * refactor(file_io): implement path locks cache eviction mechanism - Add _PATH_LOCKS_MAX constant set to 1024 for cache size limit - Implement cache eviction logic when locks exceed maximum capacity - Remove half of unlocked entries when cache limit is reached - Use list comprehension to identify unlocked locks for removal - Maintain existing path normalization and locking behavior fix(edit): correct method call from public to private fail method - Change self.fail to self._fail for internal error handling - Maintain consistent private method usage within class fix(mcp_client): change pop to get for optional command and args - Replace kwargs.pop with kwargs.get to avoid removing keys - Preserve original kwargs dictionary contents - Maintain default empty string and list values feat(reme): add client backend validation with error raising - Check if client_cls is None before instantiation - Raise ValueError with descriptive message for unknown backends - Provide clear error feedback for invalid backend configurations * fix(components): move directory creation to start method - Moved component_metadata_path.mkdir call from __init__ to _start in base_keyword_index - Moved component_metadata_path.mkdir call from __init__ to _start in local_file_graph - Moved component_metadata_path.mkdir call from __init__ to _start in local_file_store - Ensures directory creation happens after component initialization - Prevents potential issues with path creation during object construction * fix(steps): replace assertions with runtime errors for app_context validation - Replace assert statements with explicit RuntimeError exceptions when app_context is None - Add descriptive error messages for better debugging when resolving components - Replace assert in resolve_component method with proper exception handling - Replace assert in get_file_parser method with proper exception handling - Maintain same functionality while improving error reporting clarity * refactor(file_io): split file IO utilities into modular components - Move daily note helpers to separate _daily_index module - Extract path validation and resolution to new _path module - Remove unused code and imports from _file_io module - Update import statements across affected modules - Introduce WikilinkHandler utility for link parsing - Replace regex-based link extraction with WikilinkHandler - Add integration JSONL files to gitignore - Consolidate file locking mechanism in _file_io module * style(formatter): fix spacing issues in file IO and chunked file parser - Fixed whitespace around colon in slice notation in file_io.py - Corrected spacing around colon in slice notation in chunked_file_parser.py - Applied consistent formatting for array slicing operations - Improved code readability by standardizing space placement in ranges * refactor(steps): replace property-based component resolution with Ref descriptor - Introduce Ref descriptor class for lazy component dependency resolution - Replace _resolve method and individual properties with Ref descriptors - Add as_llm, as_llm_formatter, as_token_counter, file_store, and embedding Ref attributes - Remove legacy property methods and resolve logic from BaseStep - Add cache clearing mechanism for Ref values during step calls - Update UpdateCatalogStep to use Ref instead of property-based resolution
84 lines
3.8 KiB
Python
84 lines
3.8 KiB
Python
"""Update file catalog with a batch of file changes."""
|
|
|
|
from pathlib import Path
|
|
|
|
from watchfiles import Change
|
|
|
|
from ..base_step import BaseStep, Ref
|
|
from ...components import R
|
|
from ...components.file_catalog import BaseFileCatalog
|
|
from ...enumeration import ComponentEnum
|
|
from ...schema import FileNode
|
|
|
|
|
|
@R.register("update_catalog_step")
|
|
class UpdateCatalogStep(BaseStep):
|
|
"""Classify raw watcher changes and update the file_catalog."""
|
|
|
|
file_catalog: BaseFileCatalog = Ref(BaseFileCatalog, ComponentEnum.FILE_CATALOG)
|
|
|
|
async def execute(self):
|
|
assert self.context is not None
|
|
# Each item: {"change": Change | "added"|"modified"|"deleted", "path": absolute path}
|
|
changes: list[dict] = self.context.get("changes") or []
|
|
persist: bool = bool(self.context.get("persist", False))
|
|
|
|
buckets: dict[Change, list[str]] = {Change.added: [], Change.modified: [], Change.deleted: []}
|
|
for item in changes:
|
|
c = item["change"]
|
|
if isinstance(c, str):
|
|
c = Change.__members__.get(c)
|
|
if isinstance(c, Change) and c in buckets:
|
|
buckets[c].append(item["path"])
|
|
|
|
results: list[dict] = []
|
|
|
|
for change, action in ((Change.added, "Adding"), (Change.modified, "Updating")):
|
|
paths = buckets[change]
|
|
if not paths:
|
|
continue
|
|
self.logger.info(f"Detected {len(paths)} {change.name} file(s)")
|
|
nodes: list[FileNode] = []
|
|
ok_paths: list[str] = []
|
|
for path in paths:
|
|
abs_path = Path(path)
|
|
if not abs_path.is_file():
|
|
results.append({"change": change.name, "path": path, "success": False, "error": "not a file"})
|
|
continue
|
|
self.logger.info(f"{action} file: {path}")
|
|
try:
|
|
stat = abs_path.stat()
|
|
nodes.append(FileNode(path=self.to_vault_relative(abs_path), st_mtime=stat.st_mtime))
|
|
ok_paths.append(path)
|
|
except Exception as e:
|
|
self.logger.exception(f"Failed to stat {path}")
|
|
results.append({"change": change.name, "path": path, "success": False, "error": str(e)})
|
|
if nodes:
|
|
try:
|
|
await self.file_catalog.delete([n.path for n in nodes])
|
|
await self.file_catalog.upsert(nodes)
|
|
results.extend({"change": change.name, "path": p, "success": True} for p in ok_paths)
|
|
except Exception as e:
|
|
self.logger.exception(f"Failed to upsert {len(nodes)} {change.name} file(s)")
|
|
results.extend(
|
|
{"change": change.name, "path": p, "success": False, "error": str(e)} for p in ok_paths
|
|
)
|
|
|
|
if deleted := buckets[Change.deleted]:
|
|
if self.file_catalog is None:
|
|
raise RuntimeError("file_catalog is not initialized!")
|
|
self.logger.info(f"Detected {len(deleted)} deleted file(s)")
|
|
rel_deleted = [self.to_vault_relative(p) for p in deleted]
|
|
try:
|
|
await self.file_catalog.delete(rel_deleted)
|
|
results.extend({"change": "deleted", "path": p, "success": True} for p in deleted)
|
|
except Exception as e:
|
|
self.logger.exception(f"Failed to delete {len(deleted)} file(s)")
|
|
results.extend({"change": "deleted", "path": p, "success": False, "error": str(e)} for p in deleted)
|
|
|
|
if persist and results:
|
|
await self.file_catalog.dump()
|
|
|
|
self.context.response.answer = results
|
|
self.context.response.success = all(r["success"] for r in results) if results else True
|
|
return self.context.response
|