Merge pull request #101 from agentscope-ai/dev_0205

refactor(core): update config parsing and memory management system

.gitignore

@@ -41,4 +41,5 @@ meta_memory/*
 *.sqlite3
 **/data/*.json
 *.db
-memories/*
+memories/*
+.reme/*

benchmark/halumem/eval_reme.py

@@ -220,7 +220,7 @@ async def answer_question_with_memories(
 
     result = await reme.llm.simple_request_for_json(
         prompt=prompt,
-        model_name=model_name
+        model_name=model_name,
     )
 
     return result

reme/agent/memory/personal/personal_v1_retriever.yaml

@@ -20,7 +20,6 @@ user_message: |
     * Entity-focused queries (extract and search specific names, places, events)
     * Keyword-based searches (core concepts, topics)
     * Related context queries (broader themes)
-  - Review all results before proceeding to next phase
 
   ### Phase 2(Optional): Temporal Search
   **Tool**: `retrieve_memory` (with time filter)
@@ -32,8 +31,7 @@ user_message: |
   - After date: `20200101,99999999` (from 20200101 onwards)
   **Approach**:
   - Identify temporal constraints from the user question
-  - Refine Phase 1 queries with appropriate time filters
-  - Try multiple time ranges if initial searches yield no results
+  - Refine Phase 1 queries with 3-5 diverse appropriate different time filters
 
   ### Phase 3: Deep Dive into History
   **Tool**: `read_history`
@@ -57,6 +55,4 @@ user_message: |
   - If you find sufficient information to answer the user's question, you may output directly without exhausting all search phases
   - Exhaust all search strategies before concluding information doesn't exist
 
-  ### Output any tangentially related findings, Format:
-  [timestamp] [memory/profile/history] [relevant content1]
-  [timestamp] [memory/profile/history] [relevant content2]
+  Output a summary of all retrieved memories, user profile, and history data.

reme/agent/memory/personal/personal_v1_summarizer.yaml

@@ -15,65 +15,41 @@ user_message_s1: |
   - Extract all important information comprehensively—do not miss critical details, but avoid any fabrications or unfounded assumptions
   - The tool will retrieve similar historical memories via vector search to help you in Step 2
 
-  ### Step 2: Update and Add Memories
-  Review each memory draft from Step 1 and compare it with the retrieved historical memories, then use `update_memory` to manage all memories in one call:
+  ### Step 2: Add New Memories
+  Review each memory draft from Step 1 and compare it with the retrieved historical memories, then use `add_memory` to add new memories:
 
-  **For memories_to_update** (updating existing memories):
-  - For each memory to update, fill in the required parameters:
-    * `memory_id`: ID of the historical memory to update (from retrieved memories in Step 1)
-    * `message_time`: timestamp from the conversation (e.g., '2020-01-01 00:00:00')
-    * `memory_content`: updated or consolidated memory content
-  - Update memories when:
-    * The draft contains additional information that should be merged with existing memories
-    * Historical memories need to be corrected or refined based on new information
-
-  **For memories_to_add** (adding new memories):
-  - For each new memory, fill in the required parameters:
-    * `message_time`: timestamp from the conversation (e.g., '2020-01-01 00:00:00')
-    * `memory_content`: memory content
-  - Add memories when:
-    * The draft contains completely new information not present in historical memories
-    * The information cannot be merged into any existing memory
-
-  **General Guidelines:**
+  **Parameters for each memory:**
+  - `message_time`: timestamp from the conversation (e.g., '2020-01-01 00:00:00')
+  - `memory_content`: memory content
+  **When to skip:**
   - **Skip** drafts if their content is already fully covered by historical memories (avoid redundancy)
-  - You can update and add memories in a single `update_memory` tool call
 
 user_message_s2: |
-  You are a Profile Agent responsible for managing profiles about {memory_target}.
+  You are a User Profile Agent responsible for managing user profiles about {memory_target}.
 
   ## Latest Conversation
   Format: round<index> [<timestamp>] <role/name>: <content>
   {context}
 
-  ## Current Profiles
+  ## Current User Profiles
   {profiles}
 
   ## Task
-  Analyze the Latest Conversation and use `update_profiles` to manage profiles (both updates and additions in one call):
+  Analyze the Latest Conversation and use `update_profiles` to manage user profiles (both updates and additions in one call):
 
-  **For profiles_to_update** (updating existing profiles):
+  **For profiles_to_update** (updating existing user profiles):
   - For each profile to update, fill in the required parameters:
-    * `profile_id`: ID of the profile to update (from Current Profiles)
+    * `profile_id`: ID of the profile to update (from Current User Profiles)
     * `message_time`: timestamp from the conversation (e.g., '2020-01-01 00:00:00')
-    * `profile_key`: profile key or category (e.g., 'name', 'age', 'occupation')
-    * `profile_value`: updated profile value (e.g., 'John Smith')
-  - Update profiles when:
-    * Information in the conversation conflicts with or supersedes existing profiles
-    * Profiles need to be consolidated or merged with new information
-    * Existing profile values need to be corrected or refined
+    * `profile_key`: key (e.g., 'name', 'age', 'occupation')
+    * `profile_value`: value (e.g., 'John Smith')
 
-  **For profiles_to_add** (adding new profiles):
+  **For profiles_to_add** (adding new user profiles):
   - For each new profile, fill in the required parameters:
     * `message_time`: timestamp from the conversation (e.g., '2020-01-01 00:00:00')
-    * `profile_key`: profile key or category (e.g., 'name', 'age', 'occupation')
-    * `profile_value`: profile value (e.g., 'John Smith')
-  - Add profiles when:
-    * The information represents a new distinct profile not present in Current Profiles
-    * The profile key doesn't exist in Current Profiles
-    * The information cannot be merged into existing profiles
+    * `profile_key`: key (e.g., 'name', 'age', 'occupation')
+    * `profile_value`: value (e.g., 'John Smith')
 
   **General Guidelines:**
-  - Use actual names from the conversation (e.g., "Bob") instead of generic references (e.g., "user")
-  - Extract all important information comprehensively—do not miss critical details, but avoid any fabrications or unfounded assumptions
-  - You can update and add profiles in a single tool call
+  - Avoid any fabrications or unfounded assumptions
+  - You can update and add user profiles in a single tool call

reme/config/default.yaml

@@ -22,7 +22,7 @@ llm:
     backend: openai
     model_name: qwen3-30b-a3b-instruct-2507
     request_interval: 1
-    temperature: 0.0001
+#    temperature: 0.0001
 
   qwen3_max_instruct:
     backend: openai

reme/core/application.py

@@ -4,8 +4,10 @@ import asyncio
 
 from .context import PromptHandler, ServiceContext
 from .embedding import BaseEmbeddingModel
+from .file_watcher import BaseFileWatcher
 from .flow import BaseFlow
 from .llm import BaseLLM
+from .memory_storage import BaseMemoryStore
 from .schema import Response
 from .token_counter import BaseTokenCounter
 from .utils import execute_stream_task, PydanticConfigParser
@@ -27,7 +29,9 @@ class Application:
         llm: dict | None = None,
         embedding_model: dict | None = None,
         vector_store: dict | None = None,
+        memory_store: dict | None = None,
         token_counter: dict | None = None,
+        file_watcher: dict | None = None,
         **kwargs,
     ):
         self.service_context = ServiceContext(
@@ -43,7 +47,9 @@ class Application:
             llm=llm,
             embedding_model=embedding_model,
             vector_store=vector_store,
+            memory_store=memory_store,
             token_counter=token_counter,
+            file_watcher=file_watcher,
             **kwargs,
         )
         self.prompt_handler = PromptHandler(language=self.service_context.language)
@@ -62,7 +68,9 @@ class Application:
         llm: dict | None = None,
         embedding_model: dict | None = None,
         vector_store: dict | None = None,
+        memory_store: dict | None = None,
         token_counter: dict | None = None,
+        file_watcher: dict | None = None,
         **kwargs,
     ) -> "Application":
         """Create and start an Application instance asynchronously."""
@@ -77,7 +85,9 @@ class Application:
             llm=llm,
             embedding_model=embedding_model,
             vector_store=vector_store,
+            memory_store=memory_store,
             token_counter=token_counter,
+            file_watcher=file_watcher,
             **kwargs,
         )
         await instance.start()
@@ -145,6 +155,16 @@ class Application:
         """Get the default vector store instance."""
         return self.service_context.vector_stores.get("default")
 
+    @property
+    def memory_store(self) -> BaseMemoryStore:
+        """Get the default memory store instance."""
+        return self.service_context.memory_stores.get("default")
+
+    @property
+    def file_watcher(self) -> BaseFileWatcher:
+        """Get the default file watcher instance."""
+        return self.service_context.file_watchers.get("default")
+
     @property
     def token_counter(self) -> BaseTokenCounter:
         """Get the default token counter instance."""

reme/core/context/service_context.py

@@ -15,10 +15,11 @@ if TYPE_CHECKING:
     from ..llm import BaseLLM
     from ..embedding import BaseEmbeddingModel
     from ..vector_store import BaseVectorStore
+    from ..memory_storage import BaseMemoryStore
     from ..token_counter import BaseTokenCounter
     from ..flow import BaseFlow
     from ..service import BaseService
-    from ..memory_storage import BaseMemoryStore
+    from ..file_watcher import BaseFileWatcher
 
 
 class ServiceContext(BaseContext):
@@ -38,7 +39,9 @@ class ServiceContext(BaseContext):
         llm: dict | None = None,
         embedding_model: dict | None = None,
         vector_store: dict | None = None,
+        memory_store: dict | None = None,
         token_counter: dict | None = None,
+        file_watcher: dict | None = None,
         **kwargs,
     ):
         super().__init__()
@@ -55,7 +58,9 @@ class ServiceContext(BaseContext):
             llm=llm,
             embedding_model=embedding_model,
             vector_store=vector_store,
+            memory_store=memory_store,
             token_counter=token_counter,
+            file_watcher=file_watcher,
             **kwargs,
         )
 
@@ -79,6 +84,7 @@ class ServiceContext(BaseContext):
         self.token_counters: dict[str, "BaseTokenCounter"] = {}
         self.vector_stores: dict[str, "BaseVectorStore"] = {}
         self.memory_stores: dict[str, "BaseMemoryStore"] = {}
+        self.file_watchers: dict[str, "BaseFileWatcher"] = {}
 
         self.flows: dict[str, "BaseFlow"] = {}
         self.mcp_server_mapping: dict[str, dict] = {}
@@ -100,7 +106,9 @@ class ServiceContext(BaseContext):
         llm: dict | None = None,
         embedding_model: dict | None = None,
         vector_store: dict | None = None,
+        memory_store: dict | None = None,
         token_counter: dict | None = None,
+        file_watcher: dict | None = None,
         **kwargs,
     ) -> ServiceConfig:
 
@@ -118,9 +126,7 @@ class ServiceContext(BaseContext):
                 input_args.append(f"config={config_path}")
             if args:
                 input_args.extend(args)
-            if kwargs:
-                input_args.extend([f"{k}={v}" for k, v in kwargs.items()])
-            service_config = parser.parse_args(*input_args)
+            service_config = parser.parse_args(*input_args, **kwargs)
 
         service_config.enable_logo = enable_logo
         if llm:
@@ -131,6 +137,10 @@ class ServiceContext(BaseContext):
             self._update_section_config(service_config, "token_counter", **token_counter)
         if vector_store:
             self._update_section_config(service_config, "vector_store", **vector_store)
+        if memory_store:
+            self._update_section_config(service_config, "memory_store", **memory_store)
+        if file_watcher:
+            self._update_section_config(service_config, "file_watcher", **file_watcher)
         return service_config
 
     @staticmethod
@@ -200,10 +210,25 @@ class ServiceContext(BaseContext):
             self.memory_stores[name] = R.memory_store[config.backend](
                 store_name=config.store_name,
                 embedding_model=self.embedding_models[config.embedding_model],
+                fts_enabled=config.fts_enabled,
+                snippet_max_chars=config.snippet_max_chars,
                 **config.model_extra,
             )
             await self.memory_stores[name].start()
 
+        for name, config in self.service_config.file_watcher.items():
+            self.file_watchers[name] = R.file_watcher[config.backend](
+                watch_paths=config.watch_paths,
+                suffix_filters=config.suffix_filters,
+                recursive=config.recursive,
+                debounce=config.debounce,
+                chunk_tokens=config.chunk_tokens,
+                chunk_overlap=config.chunk_overlap,
+                memory_store=self.memory_stores[config.memory_store],
+                **config.model_extra,
+            )
+            await self.file_watchers[name].start()
+
         if self.service_config.mcp_servers:
             await self.prepare_mcp_servers()
 

reme/core/file_watcher/base_file_watcher.py

@@ -25,21 +25,25 @@ class BaseFileWatcher:
     def __init__(
         self,
         watch_paths: list[str] | str,
+        suffix_filters: list[str] | None = None,
         recursive: bool = False,
         debounce: int = 500,  # Millisecond debounce
-        suffix_filters: list[str] | None = None,
-        callback: Callable[[set[tuple[Change, str]]], None | Coroutine[Any, Any, None]] | None = None,
+        chunk_tokens: int = 400,
+        chunk_overlap: int = 80,
         memory_store: BaseMemoryStore | None = None,
+        callback: Callable[[set[tuple[Change, str]]], None | Coroutine[Any, Any, None]] | None = None,
         **kwargs,
     ):
         """
         Initialize the file watcher"""
         self.watch_paths: list[str] = [watch_paths] if isinstance(watch_paths, str) else watch_paths
+        self.suffix_filters: list[str] = suffix_filters or []
         self.recursive: bool = recursive
         self.debounce: int = debounce
-        self.suffix_filters: list[str] = suffix_filters or []
-        self.callback = callback
+        self.chunk_tokens: int = chunk_tokens
+        self.chunk_overlap: int = chunk_overlap
         self.memory_store: BaseMemoryStore = memory_store
+        self.callback = callback
         self.kwargs: dict = kwargs
 
         self._stop_event = asyncio.Event()
@@ -81,6 +85,10 @@ class BaseFileWatcher:
 
     async def _watch_loop(self):
         """Core monitoring loop"""
+        if not self.watch_paths:
+            logger.warning("No watch paths specified")
+            return
+
         async for changes in awatch(
             *self.watch_paths,
             watch_filter=self.watch_filter,

reme/core/file_watcher/delta_file_watcher.py

@@ -29,7 +29,7 @@ class DeltaFileWatcher(BaseFileWatcher):
     - Delete affected old chunks and insert new chunks
     """
 
-    def __init__(self, chunk_tokens: int = 400, chunk_overlap: int = 80, overlap_lines: int = 2, **kwargs):
+    def __init__(self, overlap_lines: int = 2, **kwargs):
         """
         Initialize delta file watcher.
 
@@ -38,10 +38,7 @@ class DeltaFileWatcher(BaseFileWatcher):
             chunk_overlap: Overlap tokens between chunks
         """
         super().__init__(**kwargs)
-        self.chunk_tokens = chunk_tokens
-        self.chunk_overlap = chunk_overlap
         self.overlap_lines = overlap_lines
-
         self.dirty = False
 
     @staticmethod

reme/core/file_watcher/full_file_watcher.py

@@ -19,12 +19,10 @@ from ..utils import chunk_markdown, hash_text
 class FullFileWatcher(BaseFileWatcher):
     """Full file watcher implementation for full synchronization"""
 
-    def __init__(self, chunk_tokens: int = 400, chunk_overlap: int = 80, **kwargs):
+    def __init__(self, **kwargs):
         """
         Initialize full file watcher"""
         super().__init__(**kwargs)
-        self.chunk_tokens = chunk_tokens
-        self.chunk_overlap = chunk_overlap
         self.dirty = False
 
     @staticmethod

reme/core/schema/__init__.py

@@ -2,7 +2,6 @@
 
 from .file_metadata import FileMetadata
 from .memory_chunk import MemoryChunk
-from .memory_index_meta import MemoryIndexMeta
 from .memory_node import MemoryNode
 from .memory_search_result import MemorySearchResult
 from .message import ContentBlock, Message, Trajectory
@@ -34,7 +33,6 @@ __all__ = [
     "LLMConfig",
     "MCPConfig",
     "MemoryChunk",
-    "MemoryIndexMeta",
     "MemoryNode",
     "MemorySearchResult",
     "Message",

reme/core/schema/memory_index_meta.py

@@ -1,14 +0,0 @@
-"""Memory index metadata schema."""
-
-from typing import Optional
-
-from pydantic import BaseModel, Field
-
-
-class MemoryIndexMeta(BaseModel):
-    """Metadata for memory index configuration."""
-
-    model: str = Field(..., description="Name of the embedding model")
-    chunk_tokens: int = Field(..., description="Maximum tokens per chunk")
-    chunk_overlap: int = Field(..., description="Number of overlapping tokens between chunks")
-    vector_dims: Optional[int] = Field(default=None, description="Vector embedding dimensions")

reme/core/schema/memory_node.py

@@ -42,6 +42,7 @@ class MemoryNode(BaseModel):
         time_modified: Last modification timestamp.
         author: Author or source of this memory.
         score: Relevance or importance score.
+        vector: Vector embedding of the memory content.
         metadata: Additional metadata for extensibility.
     """
 
@@ -58,6 +59,7 @@ class MemoryNode(BaseModel):
     author: str = Field(default="", description="Author or source of the memory")
     score: float = Field(default=0, description="Relevance or importance score")
 
+    vector: list[float] | None = Field(default=None, description="Vector embedding of the memory content")
     metadata: dict[str, Any] = Field(default_factory=dict, description="Additional metadata")
 
     def _update_modified_time(self) -> "MemoryNode":
@@ -145,6 +147,7 @@ class MemoryNode(BaseModel):
         return VectorNode(
             vector_id=self.memory_id,
             content=vector_content,
+            vector=self.vector,
             metadata=metadata,
         )
 
@@ -226,5 +229,6 @@ class MemoryNode(BaseModel):
             time_modified=metadata.pop("time_modified", ""),
             author=metadata.pop("author", ""),
             score=metadata.pop("score", 0),
+            vector=node.vector,
             metadata=metadata,
         )

reme/core/schema/service_config.py

@@ -85,6 +85,8 @@ class MemoryStoreConfig(BaseModel):
     backend: str = Field(default="sqlite")
     store_name: str = Field(default="reme")
     embedding_model: str = Field(default="default")
+    fts_enabled: bool = Field(default=True)
+    snippet_max_chars: int = Field(default=700)
 
 
 class TokenCounterConfig(BaseModel):
@@ -96,6 +98,20 @@ class TokenCounterConfig(BaseModel):
     model_name: str = Field(default="")
 
 
+class FileWatcherConfig(BaseModel):
+    """Configuration for file watcher service."""
+
+    model_config = ConfigDict(extra="allow")
+
+    watch_paths: list[str] = Field(default_factory=list)
+    suffix_filters: list[str] = Field(default_factory=list)
+    recursive: bool = Field(default=False)
+    debounce: int = Field(default=500)
+    chunk_tokens: int = Field(default=400)
+    chunk_overlap: int = Field(default=80)
+    memory_store: str = Field(default="default")
+
+
 class ServiceConfig(BaseModel):
     """Root configuration schema aggregating all service-level settings and components."""
 
@@ -121,3 +137,4 @@ class ServiceConfig(BaseModel):
     vector_store: dict[str, VectorStoreConfig] = Field(default_factory=dict)
     memory_store: dict[str, MemoryStoreConfig] = Field(default_factory=dict)
     token_counter: dict[str, TokenCounterConfig] = Field(default_factory=dict)
+    file_watcher: dict[str, FileWatcherConfig] = Field(default_factory=dict)

reme/core/utils/__init__.py

@@ -3,7 +3,7 @@
 from .cache_handler import CacheHandler
 from .case_converter import snake_to_camel, camel_to_snake
 from .chunking_utils import chunk_markdown
-from .common_utils import run_coro_safely, execute_stream_task, hash_text
+from .common_utils import run_coro_safely, execute_stream_task, hash_text, cosine_similarity, batch_cosine_similarity
 from .env_utils import load_env
 from .execute_utils import exec_code, run_shell_command
 from .http_client import HttpClient
@@ -24,6 +24,8 @@ __all__ = [
     "run_coro_safely",
     "execute_stream_task",
     "hash_text",
+    "cosine_similarity",
+    "batch_cosine_similarity",
     "load_env",
     "exec_code",
     "run_shell_command",

reme/core/utils/common_utils.py

@@ -5,6 +5,7 @@ import hashlib
 from collections.abc import AsyncGenerator, Coroutine
 from typing import Any
 
+import numpy as np
 from loguru import logger
 
 from ..enumeration import ChunkEnum
@@ -94,3 +95,54 @@ def hash_text(text: str) -> str:
         Hexadecimal representation of the SHA-256 hash
     """
     return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def cosine_similarity(vec1: list[float], vec2: list[float]) -> float:
+    """Calculate the cosine similarity between two numeric vectors."""
+    if len(vec1) != len(vec2):
+        raise ValueError(f"Vectors must have same length: {len(vec1)} != {len(vec2)}")
+
+    dot_product = sum(a * b for a, b in zip(vec1, vec2))
+    magnitude1 = sum(a * a for a in vec1) ** 0.5
+    magnitude2 = sum(b * b for b in vec2) ** 0.5
+
+    if magnitude1 == 0 or magnitude2 == 0:
+        return 0.0
+
+    return dot_product / (magnitude1 * magnitude2)
+
+
+def batch_cosine_similarity(nd_array1: np.ndarray, nd_array2: np.ndarray) -> np.ndarray:
+    """Calculate cosine similarity matrix between two batches of vectors.
+
+    Args:
+        nd_array1: Matrix of shape (batch_size1, emb_size)
+        nd_array2: Matrix of shape (batch_size2, emb_size)
+
+    Returns:
+        Similarity matrix of shape (batch_size1, batch_size2) where
+        result[i, j] is the cosine similarity between nd_array1[i] and nd_array2[j]
+
+    Raises:
+        ValueError: If embedding dimensions don't match
+    """
+    if nd_array1.shape[1] != nd_array2.shape[1]:
+        raise ValueError(f"Embedding dimensions must match: {nd_array1.shape[1]} != {nd_array2.shape[1]}")
+
+    # Compute dot products: (batch_size1, emb_size) @ (emb_size, batch_size2)
+    # Result shape: (batch_size1, batch_size2)
+    dot_products = np.dot(nd_array1, nd_array2.T)
+
+    # Compute L2 norms for each vector
+    norms1 = np.linalg.norm(nd_array1, axis=1)  # Shape: (batch_size1,)
+    norms2 = np.linalg.norm(nd_array2, axis=1)  # Shape: (batch_size2,)
+
+    # Compute outer product of norms: (batch_size1, 1) @ (1, batch_size2)
+    # Result shape: (batch_size1, batch_size2)
+    norm_products = np.outer(norms1, norms2)
+
+    # Avoid division by zero
+    norm_products = np.where(norm_products == 0, 1e-10, norm_products)
+
+    # Compute cosine similarities
+    return dot_products / norm_products

reme/core/utils/logo_utils.py

@@ -81,4 +81,4 @@ def print_logo(service_config: "ServiceConfig"):
         expand=False,
     )
 
-    Console().print(Group("\n", panel, "\n"))
+    Console().print(Group("\n", panel, "\n"), justify="center")

reme/core/utils/pydantic_config_parser.py

@@ -145,19 +145,8 @@ class PydanticConfigParser:
             raise FileNotFoundError(f"config={config_path} not found")
         return config_path
 
-    def parse_args(self, *args: str) -> T:
-        """Parse CLI arguments and load configs from YAML files.
-
-        Args:
-            *args: CLI arguments in format "key=value" or "config=file.yaml".
-
-        Returns:
-            Validated Pydantic config instance.
-
-        Raises:
-            ValueError: If no config file is specified.
-            FileNotFoundError: If specified config file does not exist.
-        """
+    def parse_args(self, *args: str, **kwargs) -> T:
+        """Parse CLI arguments and load configs from YAML files."""
         configs_to_merge = [self.config_class().model_dump()]
 
         # Separate config file path from other arguments
@@ -184,6 +173,9 @@ class PydanticConfigParser:
         if filter_args:
             configs_to_merge.append(self.parse_dot_notation(filter_args))
 
+        if kwargs:
+            configs_to_merge.append(kwargs)
+
         # Merge all configs and validate
         self.config_dict = self.merge_configs(*configs_to_merge)
         return self.config_class.model_validate(self.config_dict)

reme/core/vector_store/local_vector_store.py

@@ -9,6 +9,7 @@ from loguru import logger
 from .base_vector_store import BaseVectorStore
 from ..embedding import BaseEmbeddingModel
 from ..schema import VectorNode
+from ..utils import cosine_similarity
 
 
 class LocalVectorStore(BaseVectorStore):
@@ -79,21 +80,6 @@ class LocalVectorStore(BaseVectorStore):
 
         return nodes
 
-    @staticmethod
-    def _cosine_similarity(vec1: list[float], vec2: list[float]) -> float:
-        """Calculate the cosine similarity between two numeric vectors."""
-        if len(vec1) != len(vec2):
-            raise ValueError(f"Vectors must have same length: {len(vec1)} != {len(vec2)}")
-
-        dot_product = sum(a * b for a, b in zip(vec1, vec2))
-        magnitude1 = sum(a * a for a in vec1) ** 0.5
-        magnitude2 = sum(b * b for b in vec2) ** 0.5
-
-        if magnitude1 == 0 or magnitude2 == 0:
-            return 0.0
-
-        return dot_product / (magnitude1 * magnitude2)
-
     @staticmethod
     def _match_filters(node: VectorNode, filters: dict | None) -> bool:
         """Check if a vector node matches the provided metadata filters.
@@ -208,7 +194,7 @@ class LocalVectorStore(BaseVectorStore):
                 continue
 
             try:
-                score = self._cosine_similarity(query_vector, node.vector)
+                score = cosine_similarity(query_vector, node.vector)
                 scored_nodes.append((node, score))
             except ValueError as e:
                 logger.warning(f"Failed to calculate similarity for node {node.vector_id}: {e}")

reme/reme.py

@@ -37,7 +37,7 @@ from .tool.memory import (
     AddHistory,
     ReadAllProfiles,
     UpdateProfilesV1,
-    UpdateMemoryV1,
+    AddMemory,
 )
 
 
@@ -223,7 +223,7 @@ class ReMe(Application):
                         enable_when_to_use=False,
                         enable_multiple=True,
                     ),
-                    UpdateMemoryV1(
+                    AddMemory(
                         enable_thinking_params=enable_thinking_params,
                         enable_memory_target=False,
                         enable_when_to_use=False,

reme/reme_fs.py

@@ -0,0 +1,41 @@
+"""ReMe File System"""
+
+from .config import ReMeConfigParser
+from .core import Application
+
+
+class ReMeFs(Application):
+    """ReMe File System"""
+
+    def __init__(
+        self,
+        *args,
+        llm_api_key: str | None = None,
+        llm_api_base: str | None = None,
+        embedding_api_key: str | None = None,
+        embedding_api_base: str | None = None,
+        enable_logo: bool = True,
+        llm: dict | None = None,
+        embedding_model: dict | None = None,
+        vector_store: dict | None = None,
+        token_counter: dict | None = None,
+        working_dir: str = "./agent",
+        **kwargs,
+    ):
+        """Initialize ReMe with config."""
+        super().__init__(
+            *args,
+            llm_api_key=llm_api_key,
+            llm_api_base=llm_api_base,
+            embedding_api_key=embedding_api_key,
+            embedding_api_base=embedding_api_base,
+            enable_logo=enable_logo,
+            parser=ReMeConfigParser,
+            llm=llm,
+            embedding_model=embedding_model,
+            vector_store=vector_store,
+            token_counter=token_counter,
+            **kwargs,
+        )
+
+        self.working_dir: str = working_dir

reme/tool/memory/history/read_history_v2.py

@@ -6,7 +6,7 @@ from loguru import logger
 
 from ..base_memory_tool import BaseMemoryTool
 from ....core.schema import MemoryNode, ToolCall, Message
-from ....core.utils import format_messages
+from ....core.utils import format_messages, cosine_similarity
 
 
 class ReadHistoryV2(BaseMemoryTool):
@@ -103,7 +103,7 @@ class ReadHistoryV2(BaseMemoryTool):
             for block in message_blocks:
                 block_text = format_messages(block, add_index=False)
                 block_embedding = await self.embedding_model.get_embedding(block_text)
-                similarity = self._calculate_cosine_similarity(query_embedding, block_embedding)
+                similarity = cosine_similarity(query_embedding, block_embedding)
                 block_similarities.append((similarity, block_text))
 
             block_similarities.sort(key=lambda x: x[0], reverse=True)
@@ -121,22 +121,3 @@ class ReadHistoryV2(BaseMemoryTool):
         history_ids = [item["history_id"] for item in history_items]
         logger.info(f"Successfully read {len(all_results)} history result(s): {history_ids}")
         return output
-
-    @staticmethod
-    def _calculate_cosine_similarity(vec1: list[float], vec2: list[float]) -> float:
-        """Calculate cosine similarity between two vectors"""
-        if len(vec1) != len(vec2):
-            raise ValueError(f"Vectors must have same length: {len(vec1)} != {len(vec2)}")
-
-        try:
-            dot_product = sum(a * b for a, b in zip(vec1, vec2))
-            magnitude1 = sum(a * a for a in vec1) ** 0.5
-            magnitude2 = sum(b * b for b in vec2) ** 0.5
-
-            if magnitude1 == 0 or magnitude2 == 0:
-                return 0.0
-
-            return dot_product / (magnitude1 * magnitude2)
-        except Exception as e:
-            logger.error(f"Error calculating cosine similarity: {e}")
-            return 0.0

reme/tool/memory/vector/memory_handler.py

@@ -1,8 +1,12 @@
 """Memory handler"""
 
+import numpy as np
+from loguru import logger
+
 from ....core.context import ServiceContext
 from ....core.enumeration import MemoryType
 from ....core.schema import MemoryNode
+from ....core.utils.common_utils import batch_cosine_similarity
 from ....core.vector_store import BaseVectorStore
 
 
@@ -195,17 +199,77 @@ class MemoryHandler:
 
         return list(seen_ids.values())
 
-    async def batch_search(self, searches: list[dict]) -> list[MemoryNode]:
+    async def batch_search(self, searches: list[dict], hybrid_threshold: float = None) -> list[MemoryNode]:
         """Execute multiple search queries in batch and return deduplicated results."""
-        seen_ids: dict[str, MemoryNode] = {}
+        if hybrid_threshold is not None:
+            # Extract query list from searches
+            query_list: list[str] = [search["query"] for search in searches]
 
-        for search_params in searches:
-            search_result = await self.search(**search_params)
-            for memory_node in search_result:
-                if memory_node.memory_id not in seen_ids:
-                    seen_ids[memory_node.memory_id] = memory_node
+            # Step 1: Get embeddings for all queries using the embedding model
+            # Shape: [query_size X emb_size]
+            embedding_model = self.vector_store.embedding_model
+            query_embeddings_list: list[list[float]] = await embedding_model.get_embeddings(query_list)
+            query_embeddings = np.array(query_embeddings_list)  # Convert to numpy array
 
-        return list(seen_ids.values())
+            # Step 2: Use self.search to get search results for each query and deduplicate
+            seen_ids: dict[str, MemoryNode] = {}
+            for search_params in searches:
+                search_result = await self.search(**search_params)
+                for memory_node in search_result:
+                    if memory_node.memory_id not in seen_ids:
+                        seen_ids[memory_node.memory_id] = memory_node
+
+            # Step 3: Get deduplicated results
+            deduplicated_results = list(seen_ids.values())
+
+            # If no results, return empty list
+            if not deduplicated_results:
+                return []
+
+            # Step 4: Extract embeddings from results
+            # Shape: [result_size X emb_size]
+            result_embeddings_list = [node.vector for node in deduplicated_results if node.vector]
+
+            # Filter out nodes without embeddings
+            results_with_embeddings = [node for node in deduplicated_results if node.vector]
+
+            if not result_embeddings_list:
+                logger.warning("No results with embeddings found")
+                return deduplicated_results
+
+            result_embeddings = np.array(result_embeddings_list)
+
+            # Step 5: Compute cosine similarity matrix
+            # Shape: [query_size X result_size]
+            similarity_matrix = batch_cosine_similarity(query_embeddings, result_embeddings)
+
+            # Step 6: Calculate average score for each result across all queries
+            # Shape: [result_size]
+            avg_scores = np.mean(similarity_matrix, axis=0)
+
+            # Step 7: Filter results by hybrid_threshold and sort by average score
+            filtered_results = []
+            for idx, node in enumerate(results_with_embeddings):
+                if avg_scores[idx] >= hybrid_threshold:
+                    node.score = float(avg_scores[idx])
+                    filtered_results.append(node)
+
+            # Sort by score in descending order
+            filtered_results.sort(key=lambda x: x.score, reverse=True)
+
+            return filtered_results
+
+        else:
+            # Original behavior: simple deduplication without hybrid scoring
+            seen_ids: dict[str, MemoryNode] = {}
+
+            for search_params in searches:
+                search_result = await self.search(**search_params)
+                for memory_node in search_result:
+                    if memory_node.memory_id not in seen_ids:
+                        seen_ids[memory_node.memory_id] = memory_node
+
+            return list(seen_ids.values())
 
     async def list(
         self,

reme/tool/memory/vector/retrieve_memory.py

@@ -11,11 +11,19 @@ from ....core.utils import deduplicate_memories
 class RetrieveMemory(BaseMemoryTool):
     """Tool to retrieve memories using similarity search"""
 
-    def __init__(self, top_k: int = 20, enable_memory_target: bool = False, enable_time_filter: bool = False, **kwargs):
+    def __init__(
+        self,
+        top_k: int = 20,
+        enable_memory_target: bool = False,
+        enable_time_filter: bool = False,
+        hybrid_threshold: float | None = None,
+        **kwargs,
+    ):
         super().__init__(**kwargs)
         self.top_k: int = top_k
         self.enable_memory_target: bool = enable_memory_target
         self.enable_time_filter: bool = enable_time_filter
+        self.hybrid_threshold: float | None = hybrid_threshold
 
     def _build_query_parameters(self) -> dict:
         """Build the query parameters schema based on enabled features."""
@@ -111,7 +119,11 @@ class RetrieveMemory(BaseMemoryTool):
         memory_nodes: list[MemoryNode] = []
         for target, searches in queries_by_target.items():
             handler = MemoryHandler(target, self.service_context)
-            nodes = await handler.batch_search(searches)
+            if self.hybrid_threshold is not None:
+                nodes = await handler.batch_search(searches, self.hybrid_threshold)
+                nodes = nodes[: self.top_k]
+            else:
+                nodes = await handler.batch_search(searches)
             memory_nodes.extend(nodes)
 
         memory_nodes = deduplicate_memories(memory_nodes)

tests/test_vector_store.py

@@ -26,7 +26,7 @@ from loguru import logger
 
 from reme.core.embedding import OpenAIEmbeddingModel
 from reme.core.schema import VectorNode
-from reme.core.utils import load_env
+from reme.core.utils import load_env, cosine_similarity
 from reme.core.vector_store import (
     BaseVectorStore,
     ChromaVectorStore,
@@ -657,19 +657,19 @@ async def test_cosine_similarity(store_name: str):
     vec3 = [1.0, 0.0, 0.0]
 
     # Test perpendicular vectors (similarity = 0)
-    sim1 = LocalVectorStore._cosine_similarity(vec1, vec2)  # pylint: disable=protected-access
+    sim1 = cosine_similarity(vec1, vec2)  # pylint: disable=protected-access
     logger.info(f"Similarity between perpendicular vectors: {sim1:.4f}")
     assert abs(sim1) < 0.0001, "Perpendicular vectors should have similarity close to 0"
 
     # Test identical vectors (similarity = 1)
-    sim2 = LocalVectorStore._cosine_similarity(vec1, vec3)  # pylint: disable=protected-access
+    sim2 = cosine_similarity(vec1, vec3)  # pylint: disable=protected-access
     logger.info(f"Similarity between identical vectors: {sim2:.4f}")
     assert abs(sim2 - 1.0) < 0.0001, "Identical vectors should have similarity close to 1"
 
     # Test with real-world like vectors
     vec4 = [0.5, 0.5, 0.5]
     vec5 = [0.6, 0.4, 0.5]
-    sim3 = LocalVectorStore._cosine_similarity(vec4, vec5)  # pylint: disable=protected-access
+    sim3 = cosine_similarity(vec4, vec5)  # pylint: disable=protected-access
     logger.info(f"Similarity between similar vectors: {sim3:.4f}")
     assert sim3 > 0.9, "Similar vectors should have high similarity"