From 9e50e71b52225001c011aca76cda5a09d40124d1 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Fri, 6 Feb 2026 15:35:09 +0800 Subject: [PATCH] feat(core): add memory store and file watcher support to application --- reme/core/application.py | 20 ++++++++++++++ reme/core/context/service_context.py | 29 +++++++++++++++++++- reme/core/file_watcher/base_file_watcher.py | 16 ++++++++--- reme/core/file_watcher/delta_file_watcher.py | 5 +--- reme/core/file_watcher/full_file_watcher.py | 4 +-- reme/core/schema/__init__.py | 2 -- reme/core/schema/memory_index_meta.py | 14 ---------- reme/core/schema/service_config.py | 17 ++++++++++++ 8 files changed, 79 insertions(+), 28 deletions(-) delete mode 100644 reme/core/schema/memory_index_meta.py diff --git a/reme/core/application.py b/reme/core/application.py index b6f1da43..ff093692 100644 --- a/reme/core/application.py +++ b/reme/core/application.py @@ -4,8 +4,10 @@ import asyncio from .context import PromptHandler, ServiceContext from .embedding import BaseEmbeddingModel +from .file_watcher import BaseFileWatcher from .flow import BaseFlow from .llm import BaseLLM +from .memory_storage import BaseMemoryStore from .schema import Response from .token_counter import BaseTokenCounter from .utils import execute_stream_task, PydanticConfigParser @@ -27,7 +29,9 @@ class Application: llm: dict | None = None, embedding_model: dict | None = None, vector_store: dict | None = None, + memory_store: dict | None = None, token_counter: dict | None = None, + file_watcher: dict | None = None, **kwargs, ): self.service_context = ServiceContext( @@ -43,7 +47,9 @@ class Application: llm=llm, embedding_model=embedding_model, vector_store=vector_store, + memory_store=memory_store, token_counter=token_counter, + file_watcher=file_watcher, **kwargs, ) self.prompt_handler = PromptHandler(language=self.service_context.language) @@ -62,7 +68,9 @@ class Application: llm: dict | None = None, embedding_model: dict | None = None, vector_store: dict | None = None, + memory_store: dict | None = None, token_counter: dict | None = None, + file_watcher: dict | None = None, **kwargs, ) -> "Application": """Create and start an Application instance asynchronously.""" @@ -77,7 +85,9 @@ class Application: llm=llm, embedding_model=embedding_model, vector_store=vector_store, + memory_store=memory_store, token_counter=token_counter, + file_watcher=file_watcher, **kwargs, ) await instance.start() @@ -145,6 +155,16 @@ class Application: """Get the default vector store instance.""" return self.service_context.vector_stores.get("default") + @property + def memory_store(self) -> BaseMemoryStore: + """Get the default memory store instance.""" + return self.service_context.memory_stores.get("default") + + @property + def file_watcher(self) -> BaseFileWatcher: + """Get the default file watcher instance.""" + return self.service_context.file_watchers.get("default") + @property def token_counter(self) -> BaseTokenCounter: """Get the default token counter instance.""" diff --git a/reme/core/context/service_context.py b/reme/core/context/service_context.py index 045858f7..a74eac40 100644 --- a/reme/core/context/service_context.py +++ b/reme/core/context/service_context.py @@ -15,10 +15,11 @@ if TYPE_CHECKING: from ..llm import BaseLLM from ..embedding import BaseEmbeddingModel from ..vector_store import BaseVectorStore + from ..memory_storage import BaseMemoryStore from ..token_counter import BaseTokenCounter from ..flow import BaseFlow from ..service import BaseService - from ..memory_storage import BaseMemoryStore + from ..file_watcher import BaseFileWatcher class ServiceContext(BaseContext): @@ -38,7 +39,9 @@ class ServiceContext(BaseContext): llm: dict | None = None, embedding_model: dict | None = None, vector_store: dict | None = None, + memory_store: dict | None = None, token_counter: dict | None = None, + file_watcher: dict | None = None, **kwargs, ): super().__init__() @@ -55,7 +58,9 @@ class ServiceContext(BaseContext): llm=llm, embedding_model=embedding_model, vector_store=vector_store, + memory_store=memory_store, token_counter=token_counter, + file_watcher=file_watcher, **kwargs, ) @@ -79,6 +84,7 @@ class ServiceContext(BaseContext): self.token_counters: dict[str, "BaseTokenCounter"] = {} self.vector_stores: dict[str, "BaseVectorStore"] = {} self.memory_stores: dict[str, "BaseMemoryStore"] = {} + self.file_watchers: dict[str, "BaseFileWatcher"] = {} self.flows: dict[str, "BaseFlow"] = {} self.mcp_server_mapping: dict[str, dict] = {} @@ -100,7 +106,9 @@ class ServiceContext(BaseContext): llm: dict | None = None, embedding_model: dict | None = None, vector_store: dict | None = None, + memory_store: dict | None = None, token_counter: dict | None = None, + file_watcher: dict | None = None, **kwargs, ) -> ServiceConfig: @@ -129,6 +137,10 @@ class ServiceContext(BaseContext): self._update_section_config(service_config, "token_counter", **token_counter) if vector_store: self._update_section_config(service_config, "vector_store", **vector_store) + if memory_store: + self._update_section_config(service_config, "memory_store", **memory_store) + if file_watcher: + self._update_section_config(service_config, "file_watcher", **file_watcher) return service_config @staticmethod @@ -198,10 +210,25 @@ class ServiceContext(BaseContext): self.memory_stores[name] = R.memory_store[config.backend]( store_name=config.store_name, embedding_model=self.embedding_models[config.embedding_model], + fts_enabled=config.fts_enabled, + snippet_max_chars=config.snippet_max_chars, **config.model_extra, ) await self.memory_stores[name].start() + for name, config in self.service_config.file_watcher.items(): + self.file_watchers[name] = R.file_watcher[config.backend]( + watch_paths=config.watch_paths, + suffix_filters=config.suffix_filters, + recursive=config.recursive, + debounce=config.debounce, + chunk_tokens=config.chunk_tokens, + chunk_overlap=config.chunk_overlap, + memory_store=self.memory_stores[config.memory_store], + **config.model_extra, + ) + await self.file_watchers[name].start() + if self.service_config.mcp_servers: await self.prepare_mcp_servers() diff --git a/reme/core/file_watcher/base_file_watcher.py b/reme/core/file_watcher/base_file_watcher.py index 8df6b9b4..620d2e17 100644 --- a/reme/core/file_watcher/base_file_watcher.py +++ b/reme/core/file_watcher/base_file_watcher.py @@ -25,21 +25,25 @@ class BaseFileWatcher: def __init__( self, watch_paths: list[str] | str, + suffix_filters: list[str] | None = None, recursive: bool = False, debounce: int = 500, # Millisecond debounce - suffix_filters: list[str] | None = None, - callback: Callable[[set[tuple[Change, str]]], None | Coroutine[Any, Any, None]] | None = None, + chunk_tokens: int = 400, + chunk_overlap: int = 80, memory_store: BaseMemoryStore | None = None, + callback: Callable[[set[tuple[Change, str]]], None | Coroutine[Any, Any, None]] | None = None, **kwargs, ): """ Initialize the file watcher""" self.watch_paths: list[str] = [watch_paths] if isinstance(watch_paths, str) else watch_paths + self.suffix_filters: list[str] = suffix_filters or [] self.recursive: bool = recursive self.debounce: int = debounce - self.suffix_filters: list[str] = suffix_filters or [] - self.callback = callback + self.chunk_tokens: int = chunk_tokens + self.chunk_overlap: int = chunk_overlap self.memory_store: BaseMemoryStore = memory_store + self.callback = callback self.kwargs: dict = kwargs self._stop_event = asyncio.Event() @@ -81,6 +85,10 @@ class BaseFileWatcher: async def _watch_loop(self): """Core monitoring loop""" + if not self.watch_paths: + logger.warning("No watch paths specified") + return + async for changes in awatch( *self.watch_paths, watch_filter=self.watch_filter, diff --git a/reme/core/file_watcher/delta_file_watcher.py b/reme/core/file_watcher/delta_file_watcher.py index 60807133..0c9cae7c 100644 --- a/reme/core/file_watcher/delta_file_watcher.py +++ b/reme/core/file_watcher/delta_file_watcher.py @@ -29,7 +29,7 @@ class DeltaFileWatcher(BaseFileWatcher): - Delete affected old chunks and insert new chunks """ - def __init__(self, chunk_tokens: int = 400, chunk_overlap: int = 80, overlap_lines: int = 2, **kwargs): + def __init__(self, overlap_lines: int = 2, **kwargs): """ Initialize delta file watcher. @@ -38,10 +38,7 @@ class DeltaFileWatcher(BaseFileWatcher): chunk_overlap: Overlap tokens between chunks """ super().__init__(**kwargs) - self.chunk_tokens = chunk_tokens - self.chunk_overlap = chunk_overlap self.overlap_lines = overlap_lines - self.dirty = False @staticmethod diff --git a/reme/core/file_watcher/full_file_watcher.py b/reme/core/file_watcher/full_file_watcher.py index 69870c2c..c3c08406 100644 --- a/reme/core/file_watcher/full_file_watcher.py +++ b/reme/core/file_watcher/full_file_watcher.py @@ -19,12 +19,10 @@ from ..utils import chunk_markdown, hash_text class FullFileWatcher(BaseFileWatcher): """Full file watcher implementation for full synchronization""" - def __init__(self, chunk_tokens: int = 400, chunk_overlap: int = 80, **kwargs): + def __init__(self, **kwargs): """ Initialize full file watcher""" super().__init__(**kwargs) - self.chunk_tokens = chunk_tokens - self.chunk_overlap = chunk_overlap self.dirty = False @staticmethod diff --git a/reme/core/schema/__init__.py b/reme/core/schema/__init__.py index a4a4214e..d4c5c051 100644 --- a/reme/core/schema/__init__.py +++ b/reme/core/schema/__init__.py @@ -2,7 +2,6 @@ from .file_metadata import FileMetadata from .memory_chunk import MemoryChunk -from .memory_index_meta import MemoryIndexMeta from .memory_node import MemoryNode from .memory_search_result import MemorySearchResult from .message import ContentBlock, Message, Trajectory @@ -34,7 +33,6 @@ __all__ = [ "LLMConfig", "MCPConfig", "MemoryChunk", - "MemoryIndexMeta", "MemoryNode", "MemorySearchResult", "Message", diff --git a/reme/core/schema/memory_index_meta.py b/reme/core/schema/memory_index_meta.py deleted file mode 100644 index a4538b77..00000000 --- a/reme/core/schema/memory_index_meta.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Memory index metadata schema.""" - -from typing import Optional - -from pydantic import BaseModel, Field - - -class MemoryIndexMeta(BaseModel): - """Metadata for memory index configuration.""" - - model: str = Field(..., description="Name of the embedding model") - chunk_tokens: int = Field(..., description="Maximum tokens per chunk") - chunk_overlap: int = Field(..., description="Number of overlapping tokens between chunks") - vector_dims: Optional[int] = Field(default=None, description="Vector embedding dimensions") diff --git a/reme/core/schema/service_config.py b/reme/core/schema/service_config.py index 16a70b3e..cf7da9d6 100644 --- a/reme/core/schema/service_config.py +++ b/reme/core/schema/service_config.py @@ -85,6 +85,8 @@ class MemoryStoreConfig(BaseModel): backend: str = Field(default="sqlite") store_name: str = Field(default="reme") embedding_model: str = Field(default="default") + fts_enabled: bool = Field(default=True) + snippet_max_chars: int = Field(default=700) class TokenCounterConfig(BaseModel): @@ -96,6 +98,20 @@ class TokenCounterConfig(BaseModel): model_name: str = Field(default="") +class FileWatchConfig(BaseModel): + """Configuration for file watch service.""" + + model_config = ConfigDict(extra="allow") + + watch_paths: list[str] = Field(default_factory=list) + suffix_filters: list[str] = Field(default_factory=list) + recursive: bool = Field(default=False) + debounce: int = Field(default=500) + chunk_tokens: int = Field(default=400) + chunk_overlap: int = Field(default=80) + memory_store: str = Field(default="default") + + class ServiceConfig(BaseModel): """Root configuration schema aggregating all service-level settings and components.""" @@ -121,3 +137,4 @@ class ServiceConfig(BaseModel): vector_store: dict[str, VectorStoreConfig] = Field(default_factory=dict) memory_store: dict[str, MemoryStoreConfig] = Field(default_factory=dict) token_counter: dict[str, TokenCounterConfig] = Field(default_factory=dict) + file_watch: dict[str, FileWatchConfig] = Field(default_factory=dict)