scriptoriumadmin/ReMe
1
0
Fork 0
mirror of https://github.com/agentscope-ai/ReMe.git synced 2026-10-09 03:20:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(core): add memory store and file watcher support to application

Browse source
This commit is contained in:
jinli.yl 2026-02-06 15:35:09 +08:00
parent 1dd81f9c25
commit 9e50e71b52
8 changed files with 79 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
reme/core/application.py
View file

@ -4,8 +4,10 @@ import asyncio
from .context import PromptHandler, ServiceContext
from .embedding import BaseEmbeddingModel
from .file_watcher import BaseFileWatcher
from .flow import BaseFlow
from .llm import BaseLLM
from .memory_storage import BaseMemoryStore
from .schema import Response
from .token_counter import BaseTokenCounter
from .utils import execute_stream_task, PydanticConfigParser
@ -27,7 +29,9 @@ class Application:
llm: dict | None = None,
embedding_model: dict | None = None,
vector_store: dict | None = None,
memory_store: dict | None = None,
token_counter: dict | None = None,
file_watcher: dict | None = None,
**kwargs,
):
self.service_context = ServiceContext(
@ -43,7 +47,9 @@ class Application:
llm=llm,
embedding_model=embedding_model,
vector_store=vector_store,
memory_store=memory_store,
token_counter=token_counter,
file_watcher=file_watcher,
**kwargs,
)
self.prompt_handler = PromptHandler(language=self.service_context.language)
@ -62,7 +68,9 @@ class Application:
llm: dict | None = None,
embedding_model: dict | None = None,
vector_store: dict | None = None,
memory_store: dict | None = None,
token_counter: dict | None = None,
file_watcher: dict | None = None,
**kwargs,
) -> "Application":
"""Create and start an Application instance asynchronously."""
@ -77,7 +85,9 @@ class Application:
llm=llm,
embedding_model=embedding_model,
vector_store=vector_store,
memory_store=memory_store,
token_counter=token_counter,
file_watcher=file_watcher,
**kwargs,
)
await instance.start()
@ -145,6 +155,16 @@ class Application:
"""Get the default vector store instance."""
return self.service_context.vector_stores.get("default")
@property
def memory_store(self) -> BaseMemoryStore:
"""Get the default memory store instance."""
return self.service_context.memory_stores.get("default")
@property
def file_watcher(self) -> BaseFileWatcher:
"""Get the default file watcher instance."""
return self.service_context.file_watchers.get("default")
@property
def token_counter(self) -> BaseTokenCounter:
"""Get the default token counter instance."""

29
reme/core/context/service_context.py
View file

@ -15,10 +15,11 @@ if TYPE_CHECKING:
from ..llm import BaseLLM
from ..embedding import BaseEmbeddingModel
from ..vector_store import BaseVectorStore
from ..memory_storage import BaseMemoryStore
from ..token_counter import BaseTokenCounter
from ..flow import BaseFlow
from ..service import BaseService
from ..memory_storage import BaseMemoryStore
from ..file_watcher import BaseFileWatcher
class ServiceContext(BaseContext):
@ -38,7 +39,9 @@ class ServiceContext(BaseContext):
llm: dict | None = None,
embedding_model: dict | None = None,
vector_store: dict | None = None,
memory_store: dict | None = None,
token_counter: dict | None = None,
file_watcher: dict | None = None,
**kwargs,
):
super().__init__()
@ -55,7 +58,9 @@ class ServiceContext(BaseContext):
llm=llm,
embedding_model=embedding_model,
vector_store=vector_store,
memory_store=memory_store,
token_counter=token_counter,
file_watcher=file_watcher,
**kwargs,
)
@ -79,6 +84,7 @@ class ServiceContext(BaseContext):
self.token_counters: dict[str, "BaseTokenCounter"] = {}
self.vector_stores: dict[str, "BaseVectorStore"] = {}
self.memory_stores: dict[str, "BaseMemoryStore"] = {}
self.file_watchers: dict[str, "BaseFileWatcher"] = {}
self.flows: dict[str, "BaseFlow"] = {}
self.mcp_server_mapping: dict[str, dict] = {}
@ -100,7 +106,9 @@ class ServiceContext(BaseContext):
llm: dict | None = None,
embedding_model: dict | None = None,
vector_store: dict | None = None,
memory_store: dict | None = None,
token_counter: dict | None = None,
file_watcher: dict | None = None,
**kwargs,
) -> ServiceConfig:
@ -129,6 +137,10 @@ class ServiceContext(BaseContext):
self._update_section_config(service_config, "token_counter", **token_counter)
if vector_store:
self._update_section_config(service_config, "vector_store", **vector_store)
if memory_store:
self._update_section_config(service_config, "memory_store", **memory_store)
if file_watcher:
self._update_section_config(service_config, "file_watcher", **file_watcher)
return service_config
@staticmethod
@ -198,10 +210,25 @@ class ServiceContext(BaseContext):
self.memory_stores[name] = R.memory_store[config.backend](
store_name=config.store_name,
embedding_model=self.embedding_models[config.embedding_model],
fts_enabled=config.fts_enabled,
snippet_max_chars=config.snippet_max_chars,
**config.model_extra,
)
await self.memory_stores[name].start()
for name, config in self.service_config.file_watcher.items():
self.file_watchers[name] = R.file_watcher[config.backend](
watch_paths=config.watch_paths,
suffix_filters=config.suffix_filters,
recursive=config.recursive,
debounce=config.debounce,
chunk_tokens=config.chunk_tokens,
chunk_overlap=config.chunk_overlap,
memory_store=self.memory_stores[config.memory_store],
**config.model_extra,
)
await self.file_watchers[name].start()
if self.service_config.mcp_servers:
await self.prepare_mcp_servers()

16
reme/core/file_watcher/base_file_watcher.py
View file

@ -25,21 +25,25 @@ class BaseFileWatcher:
def __init__(
self,
watch_paths: list[str] | str,
suffix_filters: list[str] | None = None,
recursive: bool = False,
debounce: int = 500, # Millisecond debounce
suffix_filters: list[str] | None = None,
callback: Callable[[set[tuple[Change, str]]], None | Coroutine[Any, Any, None]] | None = None,
chunk_tokens: int = 400,
chunk_overlap: int = 80,
memory_store: BaseMemoryStore | None = None,
callback: Callable[[set[tuple[Change, str]]], None | Coroutine[Any, Any, None]] | None = None,
**kwargs,
):
"""
Initialize the file watcher"""
self.watch_paths: list[str] = [watch_paths] if isinstance(watch_paths, str) else watch_paths
self.suffix_filters: list[str] = suffix_filters or []
self.recursive: bool = recursive
self.debounce: int = debounce
self.suffix_filters: list[str] = suffix_filters or []
self.callback = callback
self.chunk_tokens: int = chunk_tokens
self.chunk_overlap: int = chunk_overlap
self.memory_store: BaseMemoryStore = memory_store
self.callback = callback
self.kwargs: dict = kwargs
self._stop_event = asyncio.Event()
@ -81,6 +85,10 @@ class BaseFileWatcher:
async def _watch_loop(self):
"""Core monitoring loop"""
if not self.watch_paths:
logger.warning("No watch paths specified")
return
async for changes in awatch(
*self.watch_paths,
watch_filter=self.watch_filter,

5
reme/core/file_watcher/delta_file_watcher.py
View file

@ -29,7 +29,7 @@ class DeltaFileWatcher(BaseFileWatcher):
- Delete affected old chunks and insert new chunks
"""
def __init__(self, chunk_tokens: int = 400, chunk_overlap: int = 80, overlap_lines: int = 2, **kwargs):
def __init__(self, overlap_lines: int = 2, **kwargs):
"""
Initialize delta file watcher.
@ -38,10 +38,7 @@ class DeltaFileWatcher(BaseFileWatcher):
chunk_overlap: Overlap tokens between chunks
"""
super().__init__(**kwargs)
self.chunk_tokens = chunk_tokens
self.chunk_overlap = chunk_overlap
self.overlap_lines = overlap_lines
self.dirty = False
@staticmethod

4
reme/core/file_watcher/full_file_watcher.py
View file

@ -19,12 +19,10 @@ from ..utils import chunk_markdown, hash_text
class FullFileWatcher(BaseFileWatcher):
"""Full file watcher implementation for full synchronization"""
def __init__(self, chunk_tokens: int = 400, chunk_overlap: int = 80, **kwargs):
def __init__(self, **kwargs):
"""
Initialize full file watcher"""
super().__init__(**kwargs)
self.chunk_tokens = chunk_tokens
self.chunk_overlap = chunk_overlap
self.dirty = False
@staticmethod

2
reme/core/schema/__init__.py
View file

@ -2,7 +2,6 @@
from .file_metadata import FileMetadata
from .memory_chunk import MemoryChunk
from .memory_index_meta import MemoryIndexMeta
from .memory_node import MemoryNode
from .memory_search_result import MemorySearchResult
from .message import ContentBlock, Message, Trajectory
@ -34,7 +33,6 @@ __all__ = [
"LLMConfig",
"MCPConfig",
"MemoryChunk",
"MemoryIndexMeta",
"MemoryNode",
"MemorySearchResult",
"Message",

14
reme/core/schema/memory_index_meta.py
View file

@ -1,14 +0,0 @@
"""Memory index metadata schema."""
from typing import Optional
from pydantic import BaseModel, Field
class MemoryIndexMeta(BaseModel):
"""Metadata for memory index configuration."""
model: str = Field(..., description="Name of the embedding model")
chunk_tokens: int = Field(..., description="Maximum tokens per chunk")
chunk_overlap: int = Field(..., description="Number of overlapping tokens between chunks")
vector_dims: Optional[int] = Field(default=None, description="Vector embedding dimensions")

17
reme/core/schema/service_config.py
View file

@ -85,6 +85,8 @@ class MemoryStoreConfig(BaseModel):
backend: str = Field(default="sqlite")
store_name: str = Field(default="reme")
embedding_model: str = Field(default="default")
fts_enabled: bool = Field(default=True)
snippet_max_chars: int = Field(default=700)
class TokenCounterConfig(BaseModel):
@ -96,6 +98,20 @@ class TokenCounterConfig(BaseModel):
model_name: str = Field(default="")
class FileWatchConfig(BaseModel):
"""Configuration for file watch service."""
model_config = ConfigDict(extra="allow")
watch_paths: list[str] = Field(default_factory=list)
suffix_filters: list[str] = Field(default_factory=list)
recursive: bool = Field(default=False)
debounce: int = Field(default=500)
chunk_tokens: int = Field(default=400)
chunk_overlap: int = Field(default=80)
memory_store: str = Field(default="default")
class ServiceConfig(BaseModel):
"""Root configuration schema aggregating all service-level settings and components."""
@ -121,3 +137,4 @@ class ServiceConfig(BaseModel):
vector_store: dict[str, VectorStoreConfig] = Field(default_factory=dict)
memory_store: dict[str, MemoryStoreConfig] = Field(default_factory=dict)
token_counter: dict[str, TokenCounterConfig] = Field(default_factory=dict)
file_watch: dict[str, FileWatchConfig] = Field(default_factory=dict)