From 9663ee3dbcc4d69931b501a6f64040f66366afae Mon Sep 17 00:00:00 2001
From: jinliyl <6469360+jinliyl@users.noreply.github.com>
Date: Fri, 10 Apr 2026 19:20:55 +0800
Subject: [PATCH 01/16] Update references from CoPaw to QwenPaw in README
---
README.md | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/README.md b/README.md
index 781f5412..f2f413c0 100644
--- a/README.md
+++ b/README.md
@@ -33,7 +33,7 @@
| Date | Title |
|------------|-----------------------------------------------------------------|
-| 2026-03-30 | [CoPaw Context Management Design](docs/copaw_context_design.md) |
+| 2026-03-30 | [Context Management Design](docs/copaw_context_design.md) |
---
@@ -53,7 +53,7 @@ ReMe achieves state-of-the-art results on the LoCoMo and HaluMem benchmarks; see
-- **Personal assistant**: Provide long-term memory for agents like [CoPaw](https://github.com/agentscope-ai/CoPaw),
+- **Personal assistant**: Provide long-term memory for agents like [QwenPaw](https://github.com/agentscope-ai/CoPaw),
remembering user preferences and conversation history.
- **Coding assistant**: Record code style preferences and project context, maintaining a consistent development
experience across sessions.
@@ -73,7 +73,7 @@ ReMe achieves state-of-the-art results on the LoCoMo and HaluMem benchmarks; see
> Memory as files, files as memory.
Treat **memory as files** — readable, editable, and copyable.
-[CoPaw](https://github.com/agentscope-ai/CoPaw) integrates long-term memory and context management by inheriting from
+[QwenPaw](https://github.com/agentscope-ai/CoPaw) integrates long-term memory and context management by inheriting from
`ReMeLight`.
| Traditional memory system | File-based ReMe |
@@ -245,7 +245,7 @@ flowchart TD
---
-[CoPaw MemoryManager](https://github.com/agentscope-ai/CoPaw/blob/main/src/copaw/agents/memory/reme_light_memory_manager.py)
+[MemoryManager](https://github.com/agentscope-ai/CoPaw/blob/main/src/copaw/agents/memory/reme_light_memory_manager.py)
inherits `ReMeLight` and integrates its memory capabilities into the agent reasoning loop:
```mermaid
@@ -671,7 +671,7 @@ For more details on how to reproduce the experiments, see [quickstart.md](benchm
- **Need a new feature?** Open a feature request; we’ll evolve ReMe together with the community.
- **Code contributions**: All forms of contributions are welcome. Please see
the [contribution guide](docs/contribution.md).
-- **Acknowledgements**: We thank excellent open-source projects such as OpenClaw, Mem0, MemU, and CoPaw for their
+- **Acknowledgements**: We thank excellent open-source projects such as OpenClaw, Mem0, MemU, and QwenPaw for their
inspiration and support.
### Contributors
From 625d184ca12cb4bc2be69618b37b817b50febd07 Mon Sep 17 00:00:00 2001
From: Zhouwk <57825291+nitwtog@users.noreply.github.com>
Date: Mon, 13 Apr 2026 10:49:26 +0800
Subject: [PATCH 02/16] =?UTF-8?q?=E6=B7=BB=E5=8A=A0log=5Fto=5Ffile?=
=?UTF-8?q?=E7=9A=84=E5=BC=80=E5=85=B3=20(#205)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
benchmark/halumem/quickstart.md | 19 ++++++++++--
reme/core/application.py | 7 ++++-
reme/core/schema/service_config.py | 1 +
reme/core/service_context.py | 2 ++
reme/core/utils/logger_utils.py | 49 +++++++++++++++++-------------
reme/reme.py | 2 ++
6 files changed, 55 insertions(+), 25 deletions(-)
diff --git a/benchmark/halumem/quickstart.md b/benchmark/halumem/quickstart.md
index 59dc68fa..4ef30846 100644
--- a/benchmark/halumem/quickstart.md
+++ b/benchmark/halumem/quickstart.md
@@ -14,17 +14,30 @@ conda activate ./reme-env
pip install .
```
-### 2. Clone the Repository
+### 2. Download the Dataset
```bash
cd ./benchmark/halumem
-git clone https://github.com/MemTensor/HaluMem.git
+mkdir -p data
+curl -L "https://huggingface.co/datasets/IAAR-Shanghai/HaluMem/resolve/main/HaluMem-Medium.jsonl?download=true" -o data/HaluMem-Medium.jsonl
+curl -L "https://huggingface.co/datasets/IAAR-Shanghai/HaluMem/resolve/main/HaluMem-Long.jsonl?download=true" -o data/HaluMem-Long.jsonl
+```
+
+Dataset page:
+https://huggingface.co/datasets/IAAR-Shanghai/HaluMem/tree/main
+
+If the official source is slow or inaccessible in mainland China, you can use a mirror:
+```bash
+cd ./benchmark/halumem
+mkdir -p data
+curl -L "https://hf-mirror.com/datasets/IAAR-Shanghai/HaluMem/resolve/main/HaluMem-Medium.jsonl?download=true" -o data/HaluMem-Medium.jsonl
+curl -L "https://hf-mirror.com/datasets/IAAR-Shanghai/HaluMem/resolve/main/HaluMem-Long.jsonl?download=true" -o data/HaluMem-Long.jsonl
```
### 3. Run Experiments
Launch the ReMe service to enable memory library functionality:
```bash
clear && python benchmark/halumem/eval_reme.py \
- --data_path benchmark/halumem/HaluMem/data/HaluMem-Medium.jsonl \
+ --data_path benchmark/halumem/data/HaluMem-Medium.jsonl \
--reme_model_name gpt-4o-mini-2024-07-18 \
--eval_model_name gpt-4o-mini-2024-07-18 \
--batch_size 40 \
diff --git a/reme/core/application.py b/reme/core/application.py
index 8b5b1932..042b2e80 100644
--- a/reme/core/application.py
+++ b/reme/core/application.py
@@ -44,6 +44,7 @@ class Application:
config_path: str | None = None,
enable_logo: bool = True,
log_to_console: bool = True,
+ log_to_file: bool = True,
enable_load_env: bool = True,
parser: type[PydanticConfigParser] | None = None,
default_as_llm_config: dict | None = None,
@@ -73,6 +74,7 @@ class Application:
config_path=config_path,
enable_logo=enable_logo,
log_to_console=log_to_console,
+ log_to_file=log_to_file,
default_as_llm_config=default_as_llm_config,
default_as_llm_formatter_config=default_as_llm_formatter_config,
default_llm_config=default_llm_config,
@@ -144,7 +146,10 @@ class Application:
logger.warning("Application has already started.")
return self
- init_logger(log_to_console=self.service_config.log_to_console)
+ init_logger(
+ log_to_console=self.service_config.log_to_console,
+ log_to_file=self.service_config.log_to_file,
+ )
logger.info(f"Init ReMe with config: {self.service_config.model_dump_json()}")
working_path = Path(self.service_config.working_dir)
diff --git a/reme/core/schema/service_config.py b/reme/core/schema/service_config.py
index 48b97e0a..1f8a6816 100644
--- a/reme/core/schema/service_config.py
+++ b/reme/core/schema/service_config.py
@@ -122,6 +122,7 @@ class ServiceConfig(BasicConfig):
)
ray_max_workers: int = Field(default=-1)
log_to_console: bool = Field(default=True)
+ log_to_file: bool = Field(default=True)
disabled_flows: list[str] = Field(default_factory=list)
enabled_flows: list[str] = Field(default_factory=list)
diff --git a/reme/core/service_context.py b/reme/core/service_context.py
index 2c58fe7d..5e57c9e4 100644
--- a/reme/core/service_context.py
+++ b/reme/core/service_context.py
@@ -34,6 +34,7 @@ class ServiceContext(BaseDict):
config_path: str | None = None,
enable_logo: bool = True,
log_to_console: bool = True,
+ log_to_file: bool = True,
default_as_llm_config: dict | None = None,
default_as_llm_formatter_config: dict | None = None,
default_as_token_counter_config: dict | None = None,
@@ -79,6 +80,7 @@ class ServiceContext(BaseDict):
{
"enable_logo": enable_logo,
"log_to_console": log_to_console,
+ "log_to_file": log_to_file,
"working_dir": working_dir,
},
)
diff --git a/reme/core/utils/logger_utils.py b/reme/core/utils/logger_utils.py
index 512c0a42..cc141724 100644
--- a/reme/core/utils/logger_utils.py
+++ b/reme/core/utils/logger_utils.py
@@ -5,13 +5,19 @@ import sys
from datetime import datetime
-def init_logger(log_dir: str = "logs", level: str = "INFO", log_to_console: bool = True) -> None:
+def init_logger(
+ log_dir: str = "logs",
+ level: str = "INFO",
+ log_to_console: bool = True,
+ log_to_file: bool = True,
+) -> None:
"""Initialize the logger with both file and console handlers.
Args:
log_dir: Directory path for log files
level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
log_to_console: Whether to print logs to console/screen
+ log_to_file: Whether to persist logs to files under log_dir
"""
from loguru import logger
@@ -28,25 +34,26 @@ def init_logger(log_dir: str = "logs", level: str = "INFO", log_to_console: bool
)
# Try to configure file-based logging (skip if permission denied)
- try:
- # Ensure the logging directory exists
- os.makedirs(log_dir, exist_ok=True)
+ if log_to_file:
+ try:
+ # Ensure the logging directory exists
+ os.makedirs(log_dir, exist_ok=True)
- # Generate filename based on the current timestamp
- # Use dashes instead of colons for Windows compatibility
- current_ts = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
- log_filename = f"{current_ts}.log"
- log_filepath = os.path.join(log_dir, log_filename)
+ # Generate filename based on the current timestamp
+ # Use dashes instead of colons for Windows compatibility
+ current_ts = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+ log_filename = f"{current_ts}.log"
+ log_filepath = os.path.join(log_dir, log_filename)
- # Configure file-based logging with rotation and compression
- logger.add(
- log_filepath,
- level=level,
- rotation="00:00",
- retention="7 days",
- compression="zip",
- encoding="utf-8",
- format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {file}:{line} | {function} | {message}",
- )
- except Exception as e:
- logger.error(f"Error configuring file logging: {e}")
+ # Configure file-based logging with rotation and compression
+ logger.add(
+ log_filepath,
+ level=level,
+ rotation="00:00",
+ retention="7 days",
+ compression="zip",
+ encoding="utf-8",
+ format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {file}:{line} | {function} | {message}",
+ )
+ except Exception as e:
+ logger.error(f"Error configuring file logging: {e}")
diff --git a/reme/reme.py b/reme/reme.py
index 5e2aa5b2..f559ad71 100644
--- a/reme/reme.py
+++ b/reme/reme.py
@@ -46,6 +46,7 @@ class ReMe(Application):
config_path: str = "vector",
enable_logo: bool = True,
log_to_console: bool = True,
+ log_to_file: bool = True,
default_llm_config: dict | None = None,
default_embedding_model_config: dict | None = None,
default_vector_store_config: dict | None = None,
@@ -81,6 +82,7 @@ class ReMe(Application):
config_path=config_path,
enable_logo=enable_logo,
log_to_console=log_to_console,
+ log_to_file=log_to_file,
parser=ReMeConfigParser,
default_llm_config=default_llm_config,
default_embedding_model_config=default_embedding_model_config,
From e0d0e3e568e6d2163c068ad05af2cf4536c42ad2 Mon Sep 17 00:00:00 2001
From: Zhouwk <57825291+nitwtog@users.noreply.github.com>
Date: Tue, 28 Apr 2026 15:11:45 +0800
Subject: [PATCH 03/16] =?UTF-8?q?=E6=8F=90=E4=BE=9B=E6=94=AF=E6=8C=81?=
=?UTF-8?q?=E5=90=91=E9=87=8F=E6=95=B0=E6=8D=AE=E5=BA=93=E7=9A=84profile?=
=?UTF-8?q?=E5=8A=9F=E8=83=BD=20(#221)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* feat(reme): 添加配置选项以启用或禁用个人资料功能
- 在 ReMe 初始化方法中添加 enable_profile 参数，默认值为 True
- 根据 enable_profile 设置决定是否创建 profile 目录和设置 profile_dir
- 在 PersonalSummarizer 中根据 enable_profile 条件性地添加个人资料相关工具
- 在 PersonalRetriever 中根据 enable_profile 条件性地添加 ReadAllProfiles 工具
- 修改 profile_path 属性以在禁用个人资料时返回 None
- 修改 get_profile_handler 方法以在禁用个人资料时返回 None
- 为 enable_profile 参数添加文档说明其用于云向量存储场景
* refactor(benchmark): 重构LongMemEval基准测试中的ReMe实例管理
- 移除未使用的shutil导入
- 将固定的ReMe实例改为每个问题创建独立实例以实现隔离
- 更新LLM配置名称从qwen3-max-think到qwen-max-t
- 修改模型调用逻辑使用正确的model_name参数
- 添加qwen-flash和GPT-4o-mini等新模型配置
- 统一使用"User"作为用户名，通过集合名实现隔离
- 调整并发处理数从4降至1，批处理大小从10增至30
- 每个问题类型采样数从2增至4
- 添加异步上下文管理确保资源正确释放
* reformat 2 files
* refactor(benchmark): 重构长记忆评估中的模型配置
- 将原有的 eval_model_name 替换为专门的 retrieve_model_name 用于检索操作
- 添加对 qwen-max 模型配置的支持
- 更新参数解析器以支持新的检索模型参数
- 修改最大并发数默认值从 1 提升到 4
- 调整样本数量默认值从 4 减少到 1
- 统一模型参数命名规范，区分摘要、检索和评估模型
- 优化内存处理器初始化逻辑，支持独立的检索模型配置
* fix(benchmark): 移除数据路径默认值并设为必填参数
- 将LongMemEval评估脚本中的data_path参数改为必需参数
- 将HaluMem评估脚本中的data_path参数改为必需参数
- 删除了硬编码的默认文件路径配置
- 强制用户显式指定数据集文件路径以避免路径错误
* Update __init__.py
* Update __init__.py
* fix(benchmark): 修复ReMe评估中的模型配置和空值处理问题
- 移除了retrieve_memory调用中不需要的llm_config_name参数
- 修复了长字符串打印的换行格式问题
- 添加了eval_result为空时的初始化处理
- 在accuracy评估中加入了eval_model_name参数传递
* style(benchmark): 格式化模型名称打印输出
- 移除了多行字符串中的换行符和多余空格
- 将模型名称信息合并为单行连续显示
- 保持了原有的打印格式和信息完整性
* docs(readme): 更新文档添加实验结果表格
- 在英文版 README 中添加 🧪 Experiments 章节
- 添加 LoCoMo 和 HaluMem 两个基准测试的结果表格
- 在中文版 README_ZH 中添加 🧪 实验 章节
- 添加 LoCoMo 和 HaluMem 测试集的实验配置说明
- 添加完整的实验数据对比表格和评估协议说明
* docs(readme): 更新文档中的内存系统链接
- 为基于文件的记忆系统添加锚点链接
- 为基于向量库的记忆系统添加锚点链接
- 修复英文文档中的链接格式
- 修复中文文档中的链接格式和空行问题
* docs(readme): update experimental results section in documentation
- Remove outdated experimental data placeholder "Coming soon..."
- Add complete evaluation results for LoCoMo and HaluMem benchmarks
- Include detailed performance metrics tables for all memory methods
- Update experimental settings description with ReMe backbone details
- Align evaluation protocol information with LLM-as-a-Judge approach
- Maintain consistent formatting between English and Chinese documentation
* docs(benchmark): add quick start guides for halumem and longmemeval experiments
- Created HaluMem experiment quick start guide with ReMe integration setup
- Added detailed steps for installing ReMe environment using conda
- Included repository cloning instructions for HaluMem benchmark
- Provided complete command examples for running HaluMem experiments
- Created LongMeMEval quick start guide with data download procedures
- Added wget commands for downloading cleaned dataset files
- Included evaluation script instructions for computing experiment statistics
- Documented parameter configurations for different model types and batch sizes
* docs(longmemeval): update quickstart guide documentation
- Changed project name from Halumem to Longmemeval in title
- Updated description to reference Longmemeval experiments instead of Halumem
- Maintained existing ReMe integration instructions unchanged
* chore(logger): add test comment to logger configuration
- Added test comment in logger utility function
- Removed duplicate log handling by keeping the remove() call
* chore(logger): add test comment to logger configuration
- Added test comment in logger utility function
- Removed duplicate log handling by keeping the remove() call
* feat(core): add file logging capability to application
- Added log_to_file parameter to Application class constructor
- Integrated log_to_file option in logger initialization
- Updated ServiceContext to support file logging configuration
- Modified init_logger function to conditionally enable file logging
- Added log_to_file field to ServiceConfig schema
- Updated ReMe class to include file logging option
- Wrapped file logging setup in conditional check to prevent unnecessary operations
* docs(benchmark): update HaluMem quickstart guide with dataset download instructions
- Replace repository cloning with direct dataset download using curl
- Add commands to download HaluMem-Medium.jsonl and HaluMem-Long.jsonl files
- Include both official Hugging Face and mirror download sources
- Update data path reference from nested directory to local data folder
- Add dataset page link and mirror usage instructions for mainland China access
* feat(memory): add profile retrieval tool and refactor profile management
- Introduce RetrieveProfile tool for fetching specific user profiles
- Refactor ProfileHandler to support both filesystem and vector backends
- Add async methods to ProfileHandler with synchronous fallbacks
- Update PersonalRetriever to support two-stage profile and memory retrieval
- Enhance PersonalSummarizer with improved tool partitioning logic
- Add profile_backend, profile_store_name, and profile_max_capacity configuration options
- Replace direct ProfileHandler imports with get_profile_handler method
- Implement profile search functionality with dedicated prompts and workflows
- Add FileProfileBackend and VectorProfileBackend implementations
- Update base memory tool with new profile configuration parameters
* feat(profile): add custom profile collection name support
- Add profile_collection_name parameter to Application constructor
- Allow custom database collection name for vector profiles instead of default suffix
- Update profile vector store configuration logic to use custom collection name
- Modify _ensure_profile_vector_store_config to handle custom collection names
- Update docstring with detailed parameter descriptions for profile configuration options
* test(history): add single history id acceptance test for multiple mode
- Add test case to verify multiple-mode history lookup accepts a single history_id string
- Create FakeVectorStore stub with minimal implementation for ReadHistory tests
- Return requested history node from vector store mock
- Initialize ReadHistory tool with multiple mode enabled
- Add pylint disable comment for protected access to vector store property
* refactor(memory): update profile handler and vector tools with improved formatting and error handling
- Add module docstring to profiles/__init__.py
- Add pylint disable comments for no-name-in-module and missing-function-docstring
- Format long error message in ProfileHandler.sync_run method for better readability
- Reformat parameters in ProfileHandler.aadd method to separate lines
- Update model_copy call in reme.py to span multiple lines for better readability
- Format aadd_batch call in update_profile.py to span multiple lines
---
.../personal/personal_retriever.py | 130 ++++---
.../personal/personal_retriever.yaml | 33 +-
.../personal/personal_summarizer.py | 85 +++--
reme/memory/vector_tools/__init__.py | 4 +
reme/memory/vector_tools/base_memory_tool.py | 22 +-
reme/memory/vector_tools/profiles/__init__.py | 1 +
.../add_draft_and_read_all_profiles.py | 8 +-
.../vector_tools/profiles/add_profile.py | 7 +-
.../vector_tools/profiles/delete_profile.py | 7 +-
.../vector_tools/profiles/profile_handler.py | 318 +++++++++---------
.../profiles/read_all_profiles.py | 7 +-
.../vector_tools/profiles/update_profile.py | 18 +-
.../profiles/update_profiles_v1.py | 11 +-
reme/reme.py | 164 +++++++--
tests/test_reme_memory_error_handling.py | 38 ++-
15 files changed, 544 insertions(+), 309 deletions(-)
diff --git a/reme/memory/vector_based/personal/personal_retriever.py b/reme/memory/vector_based/personal/personal_retriever.py
index b8803717..8b4965f1 100644
--- a/reme/memory/vector_based/personal/personal_retriever.py
+++ b/reme/memory/vector_based/personal/personal_retriever.py
@@ -1,39 +1,19 @@
"""Personal memory retriever agent for retrieving personal memories through vector search."""
+from loguru import logger
+
from ..base_memory_agent import BaseMemoryAgent
-from ....core.enumeration import Role, MemoryType
+from ....core.enumeration import MemoryType, Role
from ....core.op import BaseTool
from ....core.schema import Message
from ....core.utils import format_messages
+_PROFILE_TOOL_NAMES: tuple[str, ...] = ("retrieve_profile", "read_all_profiles")
+_EMPTY_PROFILE_RESULTS: tuple[str, ...] = ("", "No profiles found.", "No new profiles found.")
+
class PersonalRetriever(BaseMemoryAgent):
- """Retrieve personal memories through vector search and history reading.
-
- clear && python benchmark/halumem/eval_reme.py \
- --data_path /Users/yuli/workspace/HaluMem/data/HaluMem-Medium.jsonl \
- --reme_model_name qwen3.5-plus \
- --batch_size 10000 \
- --algo_version default
-
- 📊 Question Answering (with LLM answer):
- Correct (all): 0.8537
- Hallucination (all): 0.1159
- Omission (all): 0.0305
- Correct (valid): 0.8537
- Hallucination (valid): 0.1159
- Omission (valid): 0.0305
- Valid/Total: 164/164
-
- 📊 Question Answering (with original memories):
- Correct (all): 0.9085
- Hallucination (all): 0.0671
- Omission (all): 0.0244
- Correct (valid): 0.9085
- Hallucination (valid): 0.0671
- Omission (valid): 0.0244
- Valid/Total: 164/164
- """
+ """Retrieve personal memories through vector search and history reading."""
memory_type: MemoryType = MemoryType.PERSONAL
@@ -41,36 +21,73 @@ class PersonalRetriever(BaseMemoryAgent):
super().__init__(**kwargs)
self.return_memory_nodes: bool = return_memory_nodes
- async def build_messages(self) -> list[Message]:
+ def _get_context(self) -> str:
if self.context.get("query"):
- context = self.context.query
- elif self.context.get("messages"):
- context = self.description + "\n" + format_messages(self.context.messages)
- else:
- raise ValueError("input must have either `query` or `messages`")
-
- read_all_profiles_tool: BaseTool | None = self.pop_tool("read_all_profiles")
- if read_all_profiles_tool is not None:
- all_profiles = await read_all_profiles_tool.call(
- memory_target=self.memory_target,
- service_context=self.service_context,
- )
- else:
- all_profiles = ""
+ return self.context.query.strip()
+ if self.context.get("messages"):
+ return (self.description + "\n" + format_messages(self.context.messages)).strip()
+ raise ValueError("input must have either `query` or `messages`")
+ async def _build_s1_messages(self, context: str) -> list[Message]:
return [
Message(
role=Role.USER,
content=self.prompt_format(
- prompt_name="user_message",
+ prompt_name="user_message_s1",
memory_type=self.memory_type.value,
memory_target=self.memory_target,
- user_profile=all_profiles,
- context=context.strip(),
+ context=context,
),
),
]
+ async def _build_s2_messages(self, context: str, profiles: str) -> list[Message]:
+ return [
+ Message(
+ role=Role.USER,
+ content=self.prompt_format(
+ prompt_name="user_message_s2",
+ memory_type=self.memory_type.value,
+ memory_target=self.memory_target,
+ profiles=profiles,
+ context=context,
+ ),
+ ),
+ ]
+
+ def _partition_tools(self) -> tuple[list[BaseTool], list[BaseTool]]:
+ profile_tools: list[BaseTool] = []
+ memory_tools: list[BaseTool] = []
+ for i, tool in enumerate(self.tools):
+ name = tool.tool_call.name
+ if name in _PROFILE_TOOL_NAMES:
+ profile_tools.append(tool)
+ else:
+ memory_tools.append(tool)
+ logger.info(f"[{self.__class__.__name__}] tool_call[{i}]={tool.tool_call.simple_input_dump(as_dict=False)}")
+ return profile_tools, memory_tools
+
+ @staticmethod
+ def _extract_profile_context(tools: list[BaseTool]) -> str:
+ outputs = []
+ for tool in tools:
+ response = getattr(tool, "response", None)
+ answer = getattr(response, "answer", "")
+ if answer and answer not in _EMPTY_PROFILE_RESULTS:
+ outputs.append(answer)
+ return "\n".join(outputs)
+
+ async def _run_stage(
+ self,
+ stage: str,
+ messages: list[Message],
+ tools: list[BaseTool],
+ ) -> tuple[list[BaseTool], list[Message], bool]:
+ for message in messages:
+ role = message.name or message.role
+ logger.info(f"[{self.__class__.__name__} {stage}] role={role} {message.simple_dump(as_dict=False)}")
+ return await self.react(messages, tools, stage=stage)
+
async def _acting_step(
self,
assistant_message: Message,
@@ -91,7 +108,28 @@ class PersonalRetriever(BaseMemoryAgent):
)
async def execute(self):
- result = await super().execute()
+ context = self._get_context()
+ profile_tools, memory_tools = self._partition_tools()
+
+ tools_s1: list[BaseTool] = []
+ messages_s1: list[Message] = []
+ success_s1 = True
+ profiles = ""
+ if profile_tools:
+ messages_s1 = await self._build_s1_messages(context)
+ tools_s1, messages_s1, success_s1 = await self._run_stage("s1-profile", messages_s1, profile_tools)
+ profiles = self._extract_profile_context(tools_s1)
+
+ messages_s2 = await self._build_s2_messages(context, profiles)
+ tools_s2, messages_s2, success_s2 = await self._run_stage("s2-memory", messages_s2, memory_tools)
+
+ answer = messages_s2[-1].content if success_s2 and messages_s2 else ""
+ result = {
+ "answer": answer,
+ "success": success_s1 and success_s2,
+ "messages": messages_s1 + messages_s2,
+ "tools": tools_s1 + tools_s2,
+ }
if self.return_memory_nodes:
result["answer"] = "\n".join(
[
diff --git a/reme/memory/vector_based/personal/personal_retriever.yaml b/reme/memory/vector_based/personal/personal_retriever.yaml
index 99df1374..c52ae16f 100644
--- a/reme/memory/vector_based/personal/personal_retriever.yaml
+++ b/reme/memory/vector_based/personal/personal_retriever.yaml
@@ -1,8 +1,26 @@
-user_message: |
+user_message_s1: |
+ You are a Profile Retrieval Agent specialized in finding profile information about {memory_target}.
+
+ ## User Question
+ {context}
+
+ ## Task
+ Use the available profile tool to search for profile content that is relevant to the user question.
+
+ ## Instructions
+ - If `retrieve_profile` is available, use it to search with focused profile queries derived from the question
+ - If `read_all_profiles` is available, use it to inspect the full profile list and identify relevant rows
+ - Focus on profile attributes such as identity, location, work, education, preferences, relationships, and other long-term facts
+ - Only retrieve information that is directly relevant to the user question
+ - If no relevant profile information exists, say so clearly
+
+ Output a concise summary of the relevant profile information you found.
+
+user_message_s2: |
You are a Memory Retrieval Agent specialized in retrieving {memory_type} memories about {memory_target}.
- ## User Profile
- {user_profile}
+ ## Profile Search Results
+ {profiles}
## User Question
{context}
@@ -14,11 +32,12 @@ user_message: |
**Tool**: `retrieve_memory` (without time constraints)
**Objective**: Cast a wide net to find potentially relevant memories
**Approach**:
+ - Use the profile search results above as supporting context when forming retrieval queries
- Execute 3-5 diverse search queries using different formulations:
* Original question verbatim
* Rephrased variations (different wording, synonyms)
* Entity-focused queries (extract and search specific names, places, events)
- * Keyword-based searches (core concepts, topics)
+ * Keyword-based searches (core concepts and profile facts)
* Related context queries (broader themes)
### Phase 2(Optional): Temporal Search
@@ -31,7 +50,7 @@ user_message: |
- After date: `20200101,99999999` (from 20200101 onwards)
**Approach**:
- Identify temporal constraints from the user question
- - Refine Phase 1 queries with 3-5 diverse appropriate different time filters
+ - Refine Phase 1 queries with 3-5 diverse appropriate time filters
### Phase 3: Deep Dive into History
**Tool**: `read_history`
@@ -48,11 +67,11 @@ user_message: |
- Use this to understand the full conversation surrounding a memory
## Response Guidelines
- - Base your answer EXCLUSIVELY on user profile, retrieved memories, and history data
+ - Base your answer EXCLUSIVELY on the profile search results, retrieved memories, and history data
- Never infer, assume, or hallucinate information
- Always cite sources with timestamps: `[timestamp] Memory content`
- Present conflicting information transparently with respective timestamps
- If you find sufficient information to answer the user's question, you may output directly without exhausting all search phases
- Exhaust all search strategies before concluding information doesn't exist
- Output a summary of all retrieved memories, user profile, and history data.
+ Output a summary of all retrieved memories and relevant history data.
diff --git a/reme/memory/vector_based/personal/personal_summarizer.py b/reme/memory/vector_based/personal/personal_summarizer.py
index 8e570718..89b1316f 100644
--- a/reme/memory/vector_based/personal/personal_summarizer.py
+++ b/reme/memory/vector_based/personal/personal_summarizer.py
@@ -3,13 +3,17 @@
from loguru import logger
from ..base_memory_agent import BaseMemoryAgent
-from ....core.enumeration import Role, MemoryType
+from ....core.enumeration import MemoryType, Role
from ....core.op import BaseTool
from ....core.schema import Message
+# Optional profile tools used to pre-load profile context; consumed by the
+# summarizer itself and never exposed to the stage-two ReAct loop.
+_PROFILE_CONTEXT_TOOLS: tuple[str, ...] = ("retrieve_profile", "read_all_profiles")
+
class PersonalSummarizer(BaseMemoryAgent):
- """Two-phase personal memory processor: retrieve/add memories then update profile."""
+ """Two-phase personal memory processor: add memories, then update profiles."""
memory_type: MemoryType = MemoryType.PERSONAL
@@ -62,62 +66,71 @@ class PersonalSummarizer(BaseMemoryAgent):
**kwargs,
)
- async def execute(self):
- memory_tools = []
- profile_tools = []
- read_all_profiles_tool: BaseTool | None = None
+ def _partition_tools(self) -> tuple[list[BaseTool], list[BaseTool], BaseTool | None]:
+ """Split attached tools into memory tools, profile tools, and a profile context tool."""
+ memory_tools: list[BaseTool] = []
+ profile_tools: list[BaseTool] = []
+ profile_context_tool: BaseTool | None = None
for i, tool in enumerate(self.tools):
- tool_name = tool.tool_call.name
- if tool_name == "read_all_profiles":
- read_all_profiles_tool = tool
- elif "_memory" in tool_name:
+ name = tool.tool_call.name
+ if name in _PROFILE_CONTEXT_TOOLS:
+ profile_context_tool = tool
+ elif "_memory" in name:
memory_tools.append(tool)
- elif "_profile" in tool_name:
+ elif "_profile" in name:
profile_tools.append(tool)
else:
- raise ValueError(f"[{self.__class__.__name__}] unknown tool_name={tool_name}")
+ raise ValueError(f"[{self.__class__.__name__}] unknown tool_name={name}")
logger.info(f"[{self.__class__.__name__}] tool_call[{i}]={tool.tool_call.simple_input_dump(as_dict=False)}")
+ return memory_tools, profile_tools, profile_context_tool
- stage = "s1-memory"
- messages_s1 = await self._build_s1_messages()
- for i, message in enumerate(messages_s1):
+ async def _preload_user_profile(self, tool: BaseTool | None) -> str:
+ """Invoke the profile context tool to obtain inline profile text."""
+ if tool is None:
+ return ""
+ call_kwargs: dict = {
+ "memory_target": self.memory_target,
+ "service_context": self.service_context,
+ "retrieved_nodes": self.retrieved_nodes,
+ }
+ if tool.tool_call.name == "retrieve_profile":
+ call_kwargs["query"] = self.context.history_node.content
+ return await tool.call(**call_kwargs)
+
+ async def _run_stage(
+ self,
+ stage: str,
+ messages: list[Message],
+ tools: list[BaseTool],
+ ) -> tuple[list[BaseTool], list[Message], bool]:
+ for message in messages:
role = message.name or message.role
logger.info(f"[{self.__class__.__name__} {stage}] role={role} {message.simple_dump(as_dict=False)}")
- tools_s1, messages_s1, success_s1 = await self.react(messages_s1, memory_tools, stage=stage)
+ return await self.react(messages, tools, stage=stage)
- if read_all_profiles_tool is not None:
- profiles = await read_all_profiles_tool.call(
- memory_target=self.memory_target,
- service_context=self.service_context,
- )
- else:
- profiles = ""
+ async def execute(self):
+ memory_tools, profile_tools, profile_context_tool = self._partition_tools()
+
+ messages_s1 = await self._build_s1_messages()
+ tools_s1, messages_s1, success_s1 = await self._run_stage("s1-memory", messages_s1, memory_tools)
if profile_tools:
- stage = "s2-profile"
+ profiles = await self._preload_user_profile(profile_context_tool)
messages_s2 = await self._build_s2_messages(profiles)
- for i, message in enumerate(messages_s2):
- role = message.name or message.role
- logger.info(f"[{self.__class__.__name__} {stage}] role={role} {message.simple_dump(as_dict=False)}")
- tools_s2, messages_s2, success_s2 = await self.react(messages_s2, profile_tools, stage=stage)
+ tools_s2, messages_s2, success_s2 = await self._run_stage("s2-profile", messages_s2, profile_tools)
else:
tools_s2, messages_s2, success_s2 = [], [], True
answer = (messages_s1[-1].content if success_s1 and messages_s1 else "") + (
messages_s2[-1].content if success_s2 and messages_s2 else ""
)
- success = success_s1 and success_s2
- messages = messages_s1 + messages_s2
tools = tools_s1 + tools_s2
- memory_nodes = []
- for tool in tools:
- if tool.memory_nodes:
- memory_nodes.extend(tool.memory_nodes)
+ memory_nodes = [node for tool in tools for node in (tool.memory_nodes or [])]
return {
"answer": answer,
- "success": success,
- "messages": messages,
+ "success": success_s1 and success_s2,
+ "messages": messages_s1 + messages_s2,
"tools": tools,
"memory_nodes": memory_nodes,
}
diff --git a/reme/memory/vector_tools/__init__.py b/reme/memory/vector_tools/__init__.py
index fd5e721f..9fb04923 100644
--- a/reme/memory/vector_tools/__init__.py
+++ b/reme/memory/vector_tools/__init__.py
@@ -1,5 +1,7 @@
"""memory tools"""
+# pylint: disable=no-name-in-module
+
from .base_memory_tool import BaseMemoryTool
# chunk tools
@@ -15,6 +17,7 @@ from .profiles.add_draft_and_read_all_profiles import AddDraftAndReadAllProfiles
from .profiles.add_profile import AddProfile
from .profiles.delete_profile import DeleteProfile
from .profiles.read_all_profiles import ReadAllProfiles
+from .profiles.retrieve_profile import RetrieveProfile
from .profiles.update_profile import UpdateProfile
from .profiles.update_profiles_v1 import UpdateProfilesV1
@@ -43,6 +46,7 @@ __all__ = [
"AddProfile",
"DeleteProfile",
"ReadAllProfiles",
+ "RetrieveProfile",
"UpdateProfile",
"UpdateProfilesV1",
# record tools
diff --git a/reme/memory/vector_tools/base_memory_tool.py b/reme/memory/vector_tools/base_memory_tool.py
index 3431b164..f0781bd3 100644
--- a/reme/memory/vector_tools/base_memory_tool.py
+++ b/reme/memory/vector_tools/base_memory_tool.py
@@ -3,6 +3,7 @@
from abc import ABCMeta
from pathlib import Path
+from .profiles.profile_handler import ProfileHandler
from ...core.enumeration import MemoryType
from ...core.op import BaseTool
from ...core.schema import ToolCall, MemoryNode, ToolAttr
@@ -16,12 +17,18 @@ class BaseMemoryTool(BaseTool, metaclass=ABCMeta):
enable_multiple: bool = True,
enable_thinking_params: bool = False,
profile_dir: str = "",
+ profile_backend: str = "filesystem",
+ profile_store_name: str = "profile",
+ profile_max_capacity: int = 50,
**kwargs,
):
super().__init__(**kwargs)
self.enable_multiple: bool = enable_multiple
self.enable_thinking_params: bool = enable_thinking_params
self.profile_dir: str = profile_dir
+ self.profile_backend: str = profile_backend
+ self.profile_store_name: str = profile_store_name
+ self.profile_max_capacity: int = profile_max_capacity
def _build_tool_call(self) -> ToolCall:
"""Build and return the tool call schema"""
@@ -103,6 +110,19 @@ class BaseMemoryTool(BaseTool, metaclass=ABCMeta):
return self.context.service_context.memory_target_type_mapping
@property
- def profile_path(self) -> Path:
+ def profile_path(self) -> Path | None:
"""Get the path to the profile directory for the current collection."""
+ if not self.profile_dir:
+ return None
return Path(self.profile_dir) / self.vector_store.collection_name
+
+ def get_profile_handler(self, memory_target: str) -> ProfileHandler:
+ """Build a profile handler for the current backend configuration."""
+ return ProfileHandler(
+ memory_target=memory_target,
+ profile_path=self.profile_path,
+ service_context=self.service_context,
+ profile_backend=self.profile_backend,
+ profile_store_name=self.profile_store_name,
+ max_capacity=self.profile_max_capacity,
+ )
diff --git a/reme/memory/vector_tools/profiles/__init__.py b/reme/memory/vector_tools/profiles/__init__.py
index e69de29b..150455db 100644
--- a/reme/memory/vector_tools/profiles/__init__.py
+++ b/reme/memory/vector_tools/profiles/__init__.py
@@ -0,0 +1 @@
+"""Profile memory tools."""
diff --git a/reme/memory/vector_tools/profiles/add_draft_and_read_all_profiles.py b/reme/memory/vector_tools/profiles/add_draft_and_read_all_profiles.py
index 716c4e5e..c8e4889a 100644
--- a/reme/memory/vector_tools/profiles/add_draft_and_read_all_profiles.py
+++ b/reme/memory/vector_tools/profiles/add_draft_and_read_all_profiles.py
@@ -1,8 +1,7 @@
-"""Add draft profile and read all profiles from local storage"""
+"""Add draft profile and read all profiles from the configured backend."""
from loguru import logger
-from .profile_handler import ProfileHandler
from ..base_memory_tool import BaseMemoryTool
from ....core.schema import ToolCall
@@ -92,9 +91,8 @@ class AddDraftAndReadAllProfiles(BaseMemoryTool):
continue
targets_processed.add(target)
- profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=target)
-
- profiles_str = profile_handler.read_all(add_profile_id=True)
+ profile_handler = self.get_profile_handler(target)
+ profiles_str = await profile_handler.aread_all(add_profile_id=True)
if profiles_str:
all_profiles.append(f"## Profiles for {target}:\n{profiles_str}")
diff --git a/reme/memory/vector_tools/profiles/add_profile.py b/reme/memory/vector_tools/profiles/add_profile.py
index 880bcf45..5654ebb7 100644
--- a/reme/memory/vector_tools/profiles/add_profile.py
+++ b/reme/memory/vector_tools/profiles/add_profile.py
@@ -1,8 +1,7 @@
-"""Add user profile tool"""
+"""Add user profile tool."""
from loguru import logger
-from .profile_handler import ProfileHandler
from ..base_memory_tool import BaseMemoryTool
from ....core.schema import ToolCall
@@ -40,7 +39,7 @@ class AddProfile(BaseMemoryTool):
)
async def execute(self):
- profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=self.memory_target)
+ profile_handler = self.get_profile_handler(self.memory_target)
# Get parameters
message_time = self.context.get("message_time", "")
@@ -58,7 +57,7 @@ class AddProfile(BaseMemoryTool):
}
# Add profile using ProfileHandler
- new_nodes = profile_handler.add_batch(profiles=[profile], ref_memory_id=self.history_id)
+ new_nodes = await profile_handler.aadd_batch(profiles=[profile], ref_memory_id=self.history_id)
self.memory_nodes.extend(new_nodes)
if new_nodes:
diff --git a/reme/memory/vector_tools/profiles/delete_profile.py b/reme/memory/vector_tools/profiles/delete_profile.py
index ae5bbc05..1c5f845b 100644
--- a/reme/memory/vector_tools/profiles/delete_profile.py
+++ b/reme/memory/vector_tools/profiles/delete_profile.py
@@ -1,8 +1,7 @@
-"""Delete user profile tool"""
+"""Delete user profile tool."""
from loguru import logger
-from .profile_handler import ProfileHandler
from ..base_memory_tool import BaseMemoryTool
from ....core.schema import ToolCall
@@ -32,7 +31,7 @@ class DeleteProfile(BaseMemoryTool):
)
async def execute(self):
- profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=self.memory_target)
+ profile_handler = self.get_profile_handler(self.memory_target)
# Get profile_id parameter
profile_id = self.context.get("profile_id", "")
@@ -41,7 +40,7 @@ class DeleteProfile(BaseMemoryTool):
return "No profile_id provided, operation cancelled."
# Delete profile using ProfileHandler
- success = profile_handler.delete(profile_id)
+ success = await profile_handler.adelete(profile_id)
if success:
output = f"Successfully deleted profile with ID: {profile_id}"
diff --git a/reme/memory/vector_tools/profiles/profile_handler.py b/reme/memory/vector_tools/profiles/profile_handler.py
index 3d0df400..0df3707a 100644
--- a/reme/memory/vector_tools/profiles/profile_handler.py
+++ b/reme/memory/vector_tools/profiles/profile_handler.py
@@ -1,195 +1,123 @@
-"""Profile Handler for managing user profiles in local memory"""
+"""Profile handler facade for filesystem and vector backends."""
+# pylint: disable=missing-function-docstring
+
+import asyncio
from pathlib import Path
from loguru import logger
-from ....core.enumeration import MemoryType
+from .file_profile_backend import FileProfileBackend
+from .profile_backend import BaseProfileBackend
+from .vector_profile_backend import VectorProfileBackend
+from ....core import ServiceContext
from ....core.schema import MemoryNode
-from ....core.utils import CacheHandler, deduplicate_memories
class ProfileHandler:
- """User profile CRUD handler"""
+ """User profile facade with pluggable storage backends."""
- def __init__(self, profile_path: str | Path, memory_target: str, max_capacity: int = 50):
- """init"""
- self.memory_target: str = memory_target
- self.cache_key: str = self.memory_target.replace(" ", "_").lower()
- self.cache_handler: CacheHandler = CacheHandler(profile_path)
- self.max_capacity: int = max_capacity
-
- def _load_nodes(self) -> list[MemoryNode]:
- """Load profile nodes"""
- cached_data = self.cache_handler.load(self.cache_key, auto_clean=False)
- if not cached_data:
- return []
- return [MemoryNode(**data) for data in cached_data]
-
- def _save_nodes(self, nodes: list[MemoryNode], apply_limits: bool = True):
- """Save nodes with optional deduplication and capacity enforcement"""
- if apply_limits:
- nodes = deduplicate_memories(nodes)
-
- # Enforce capacity limit by removing the oldest profiles
- if len(nodes) > self.max_capacity:
- sorted_nodes = sorted(nodes, key=lambda n: n.message_time)
- removed_count = len(sorted_nodes) - self.max_capacity
- nodes = sorted_nodes[removed_count:]
- logger.info(
- f"Capacity limit reached: removed {removed_count} oldest profiles "
- f"(kept {len(nodes)}/{self.max_capacity})",
- )
-
- nodes_data = [node.model_dump(exclude_none=True) for node in nodes]
- self.cache_handler.save(self.cache_key, nodes_data)
- logger.info(f"Saved {len(nodes)} profiles to {self.cache_key}")
-
- def delete(self, profile_id: str | list[str]) -> bool | int:
- """Delete profile by ID(s), returns True/False for single ID or count for batch delete"""
- nodes = self._load_nodes()
- original_count = len(nodes)
-
- # Batch delete mode
- if isinstance(profile_id, list):
- profile_ids_set = set(profile_id)
- nodes = [n for n in nodes if n.memory_id not in profile_ids_set]
- deleted_count = original_count - len(nodes)
-
- if deleted_count == 0:
- logger.warning(f"No profiles found to delete from {len(profile_id)} IDs")
- return 0
-
- self._save_nodes(nodes, apply_limits=False)
- logger.info(f"Batch deleted {deleted_count} profiles")
- return deleted_count
-
- # Single delete mode
- nodes = [n for n in nodes if n.memory_id != profile_id]
-
- if len(nodes) == original_count:
- logger.warning(f"Profile {profile_id} not found")
- return False
-
- self._save_nodes(nodes, apply_limits=False)
- logger.info(f"Deleted profile {profile_id}")
- return True
-
- def delete_all(self) -> int:
- """Delete all profiles, returns count deleted"""
- nodes = self._load_nodes()
- count = len(nodes)
- self._save_nodes([], apply_limits=False)
- logger.info(f"Deleted all {count} profiles")
- return count
-
- def add(self, message_time: str, profile_key: str, profile_value: str, ref_memory_id: str = "") -> MemoryNode:
- """Add new profile, returns created MemoryNode"""
- nodes = self._load_nodes()
-
- new_node = MemoryNode(
- memory_type=MemoryType.PERSONAL,
- memory_target=self.memory_target,
- when_to_use=profile_key,
- content=profile_value,
- message_time=message_time,
- ref_memory_id=ref_memory_id,
+ def __init__(
+ self,
+ memory_target: str,
+ profile_path: str | Path | None = None,
+ service_context: ServiceContext | None = None,
+ profile_backend: str = "filesystem",
+ profile_store_name: str = "profile",
+ max_capacity: int = 50,
+ ):
+ self.memory_target = memory_target
+ self.profile_backend = profile_backend
+ self.profile_store_name = profile_store_name
+ self.max_capacity = max_capacity
+ self.cache_key = self.memory_target.replace(" ", "_").lower()
+ self.backend = self._build_backend(
+ profile_path=profile_path,
+ service_context=service_context,
)
- # Remove existing nodes with the same when_to_use (profile_key)
- original_count = len(nodes)
- nodes = [n for n in nodes if n.when_to_use != profile_key]
- if len(nodes) < original_count:
- logger.info(f"Removed {original_count - len(nodes)} duplicate profile(s) with key: {profile_key}")
-
- nodes.append(new_node)
- self._save_nodes(nodes)
- logger.info(f"Added profile: {profile_key}={profile_value}")
- return new_node
-
- def add_batch(self, profiles: list[dict], ref_memory_id: str = "") -> list[MemoryNode]:
- """Add multiple profiles in batch, returns list of created MemoryNodes"""
- if not profiles:
- return []
-
- nodes = self._load_nodes()
-
- new_nodes = [
- MemoryNode(
- memory_type=MemoryType.PERSONAL,
+ def _build_backend(
+ self,
+ profile_path: str | Path | None,
+ service_context: ServiceContext | None,
+ ) -> BaseProfileBackend:
+ if self.profile_backend == "filesystem":
+ if profile_path is None:
+ raise ValueError("profile_path is required for filesystem profile backend")
+ return FileProfileBackend(
+ profile_path=profile_path,
memory_target=self.memory_target,
- when_to_use=p.get("profile_key", ""),
- content=p.get("profile_value", ""),
- message_time=p.get("message_time", ""),
- ref_memory_id=ref_memory_id,
+ max_capacity=self.max_capacity,
)
- for p in profiles
- ]
- # Remove existing nodes with the same when_to_use (profile_key)
- new_keys = {n.when_to_use for n in new_nodes}
- original_count = len(nodes)
- nodes = [n for n in nodes if n.when_to_use not in new_keys]
- if len(nodes) < original_count:
- logger.info(f"Removed {original_count - len(nodes)} duplicate profile(s) with matching keys")
+ if self.profile_backend == "vector":
+ if service_context is None:
+ raise ValueError("service_context is required for vector profile backend")
+ return VectorProfileBackend(
+ memory_target=self.memory_target,
+ service_context=service_context,
+ vector_store_name=self.profile_store_name,
+ max_capacity=self.max_capacity,
+ )
- nodes.extend(new_nodes)
- self._save_nodes(nodes)
- logger.info(f"Batch added {len(new_nodes)} profiles")
- return new_nodes
+ raise ValueError(f"Unsupported profile backend: {self.profile_backend}")
- def update(self, profile_id: str, message_time: str, profile_key: str, profile_value: str) -> MemoryNode | None:
- """Update profile by ID, returns updated node or None if not found"""
- nodes = self._load_nodes()
+ @staticmethod
+ def _run_sync(coro):
+ try:
+ asyncio.get_running_loop()
+ except RuntimeError:
+ return asyncio.run(coro)
+ raise RuntimeError(
+ "Synchronous profile access is not available in an active event loop. Use async methods instead.",
+ )
- target_node = None
- for node in nodes:
- if node.memory_id == profile_id:
- node.when_to_use = profile_key
- node.content = profile_value
- node.message_time = message_time
- target_node = node
- break
+ async def adelete(self, profile_id: str | list[str]) -> bool | int:
+ return await self.backend.delete(profile_id)
- if target_node is None:
- logger.warning(f"Profile {profile_id} not found")
- return None
+ async def adelete_all(self) -> int:
+ return await self.backend.delete_all()
- self._save_nodes(nodes, apply_limits=False)
- logger.info(f"Updated profile {profile_id}: {profile_key}={profile_value}")
- return target_node
+ async def aadd(
+ self,
+ message_time: str,
+ profile_key: str,
+ profile_value: str,
+ ref_memory_id: str = "",
+ ) -> MemoryNode:
+ return await self.backend.add(message_time, profile_key, profile_value, ref_memory_id)
- def get_by(self, *, profile_id: str | None = None, profile_key: str | None = None) -> MemoryNode | None:
- """Get profile by ID or key"""
- if not profile_id and not profile_key:
- raise ValueError("Must provide either profile_id or profile_key")
+ async def aadd_batch(self, profiles: list[dict], ref_memory_id: str = "") -> list[MemoryNode]:
+ return await self.backend.add_batch(profiles, ref_memory_id)
- nodes = self._load_nodes()
- for node in nodes:
- if profile_id and node.memory_id == profile_id:
- return node
- if profile_key and node.when_to_use == profile_key:
- return node
- return None
+ async def aupdate(
+ self,
+ profile_id: str,
+ message_time: str,
+ profile_key: str,
+ profile_value: str,
+ ) -> MemoryNode | None:
+ return await self.backend.update(profile_id, message_time, profile_key, profile_value)
- def get_by_id(self, profile_id: str) -> MemoryNode | None:
- """Get profile by ID (convenience method)"""
- return self.get_by(profile_id=profile_id)
+ async def aget_by(self, *, profile_id: str | None = None, profile_key: str | None = None) -> MemoryNode | None:
+ return await self.backend.get_by(profile_id=profile_id, profile_key=profile_key)
- def get_by_key(self, profile_key: str) -> MemoryNode | None:
- """Get profile by key (convenience method)"""
- return self.get_by(profile_key=profile_key)
+ async def aget_by_id(self, profile_id: str) -> MemoryNode | None:
+ return await self.aget_by(profile_id=profile_id)
- def get_all(self) -> list[MemoryNode]:
- """Get all profiles, sorted by message_time"""
- nodes = self._load_nodes()
- nodes.sort(key=lambda n: n.message_time)
- return nodes
+ async def aget_by_key(self, profile_key: str) -> MemoryNode | None:
+ return await self.aget_by(profile_key=profile_key)
+
+ async def aget_all(self) -> list[MemoryNode]:
+ return await self.backend.get_all()
+
+ async def asearch(self, query: str | list[str], limit: int = 5) -> list[MemoryNode]:
+ return await self.backend.search(query=query, limit=limit)
@staticmethod
def _format_node(node: MemoryNode, add_profile_id: bool = False, add_history_id: bool = False) -> str:
- """Format a single node to string"""
parts = []
+ profile_key = str(node.metadata.get("profile_key", node.when_to_use))
if add_profile_id:
parts.append(f"profile_id={node.memory_id}")
@@ -197,16 +125,70 @@ class ProfileHandler:
if node.message_time:
parts.append(f"[{node.message_time}]")
- parts.append(f"{node.when_to_use}: {node.content}")
+ parts.append(f"{profile_key}: {node.content}")
- if add_history_id:
+ if add_history_id and node.ref_memory_id:
parts.append(f"history_id={node.ref_memory_id}")
return " ".join(parts)
- def read_all(self, add_profile_id: bool = False, add_history_id: bool = False) -> str:
- """Read all profiles and return formatted string"""
- nodes = self.get_all()
+ async def aread_all(self, add_profile_id: bool = False, add_history_id: bool = False) -> str:
+ nodes = await self.aget_all()
formatted_profiles = [self._format_node(node, add_profile_id, add_history_id) for node in nodes]
logger.info(f"Read {len(formatted_profiles)} profiles from {self.cache_key}")
return "\n".join(formatted_profiles).strip()
+
+ async def aretrieve(
+ self,
+ query: str | list[str],
+ limit: int = 5,
+ add_profile_id: bool = True,
+ add_history_id: bool = False,
+ ) -> tuple[list[MemoryNode], str]:
+ nodes = await self.asearch(query=query, limit=limit)
+ formatted_profiles = [self._format_node(node, add_profile_id, add_history_id) for node in nodes]
+ return nodes, "\n".join(formatted_profiles).strip()
+
+ def delete(self, profile_id: str | list[str]) -> bool | int:
+ if isinstance(self.backend, FileProfileBackend):
+ return self.backend.delete_sync(profile_id)
+ return self._run_sync(self.adelete(profile_id))
+
+ def delete_all(self) -> int:
+ if isinstance(self.backend, FileProfileBackend):
+ return self.backend.delete_all_sync()
+ return self._run_sync(self.adelete_all())
+
+ def add(self, message_time: str, profile_key: str, profile_value: str, ref_memory_id: str = "") -> MemoryNode:
+ if isinstance(self.backend, FileProfileBackend):
+ return self.backend.add_sync(message_time, profile_key, profile_value, ref_memory_id)
+ return self._run_sync(self.aadd(message_time, profile_key, profile_value, ref_memory_id))
+
+ def add_batch(self, profiles: list[dict], ref_memory_id: str = "") -> list[MemoryNode]:
+ if isinstance(self.backend, FileProfileBackend):
+ return self.backend.add_batch_sync(profiles, ref_memory_id)
+ return self._run_sync(self.aadd_batch(profiles, ref_memory_id))
+
+ def update(self, profile_id: str, message_time: str, profile_key: str, profile_value: str) -> MemoryNode | None:
+ if isinstance(self.backend, FileProfileBackend):
+ return self.backend.update_sync(profile_id, message_time, profile_key, profile_value)
+ return self._run_sync(self.aupdate(profile_id, message_time, profile_key, profile_value))
+
+ def get_by(self, *, profile_id: str | None = None, profile_key: str | None = None) -> MemoryNode | None:
+ if isinstance(self.backend, FileProfileBackend):
+ return self.backend.get_by_sync(profile_id=profile_id, profile_key=profile_key)
+ return self._run_sync(self.aget_by(profile_id=profile_id, profile_key=profile_key))
+
+ def get_by_id(self, profile_id: str) -> MemoryNode | None:
+ return self._run_sync(self.aget_by_id(profile_id))
+
+ def get_by_key(self, profile_key: str) -> MemoryNode | None:
+ return self._run_sync(self.aget_by_key(profile_key))
+
+ def get_all(self) -> list[MemoryNode]:
+ if isinstance(self.backend, FileProfileBackend):
+ return self.backend.get_all_sync()
+ return self._run_sync(self.aget_all())
+
+ def read_all(self, add_profile_id: bool = False, add_history_id: bool = False) -> str:
+ return self._run_sync(self.aread_all(add_profile_id, add_history_id))
diff --git a/reme/memory/vector_tools/profiles/read_all_profiles.py b/reme/memory/vector_tools/profiles/read_all_profiles.py
index 1d7ddb1a..50eab684 100644
--- a/reme/memory/vector_tools/profiles/read_all_profiles.py
+++ b/reme/memory/vector_tools/profiles/read_all_profiles.py
@@ -1,8 +1,7 @@
-"""Read user profile tool"""
+"""Read user profile tool."""
from loguru import logger
-from .profile_handler import ProfileHandler
from ..base_memory_tool import BaseMemoryTool
from ....core.schema import ToolCall
@@ -44,8 +43,8 @@ class ReadAllProfiles(BaseMemoryTool):
else:
target = self.memory_target
- profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=target)
- profiles_str = profile_handler.read_all(add_profile_id=True)
+ profile_handler = self.get_profile_handler(target)
+ profiles_str = await profile_handler.aread_all(add_profile_id=True)
if not profiles_str:
output = "No profiles found."
logger.info(output)
diff --git a/reme/memory/vector_tools/profiles/update_profile.py b/reme/memory/vector_tools/profiles/update_profile.py
index 37512714..04c932bf 100644
--- a/reme/memory/vector_tools/profiles/update_profile.py
+++ b/reme/memory/vector_tools/profiles/update_profile.py
@@ -1,8 +1,7 @@
-"""Update user profile tool"""
+"""Update user profile tool."""
from loguru import logger
-from .profile_handler import ProfileHandler
from ..base_memory_tool import BaseMemoryTool
from ....core.schema import ToolCall
@@ -82,8 +81,8 @@ class UpdateProfile(BaseMemoryTool):
# Delete profiles (using self.memory_target)
if profile_ids_to_delete:
- profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=self.memory_target)
- removed_count = profile_handler.delete(profile_ids_to_delete)
+ profile_handler = self.get_profile_handler(self.memory_target)
+ removed_count = await profile_handler.adelete(profile_ids_to_delete)
# Add new profiles
if profiles_to_add:
@@ -98,14 +97,17 @@ class UpdateProfile(BaseMemoryTool):
# Add profiles for each target
for target, target_profiles in profiles_by_target.items():
- profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=target)
- new_nodes = profile_handler.add_batch(profiles=target_profiles, ref_memory_id=self.history_id)
+ profile_handler = self.get_profile_handler(target)
+ new_nodes = await profile_handler.aadd_batch(
+ profiles=target_profiles,
+ ref_memory_id=self.history_id,
+ )
self.memory_nodes.extend(new_nodes)
added_count += len(new_nodes)
else:
# Use self.memory_target for all profiles
- profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=self.memory_target)
- new_nodes = profile_handler.add_batch(profiles=profiles_to_add, ref_memory_id=self.history_id)
+ profile_handler = self.get_profile_handler(self.memory_target)
+ new_nodes = await profile_handler.aadd_batch(profiles=profiles_to_add, ref_memory_id=self.history_id)
self.memory_nodes.extend(new_nodes)
added_count = len(new_nodes)
diff --git a/reme/memory/vector_tools/profiles/update_profiles_v1.py b/reme/memory/vector_tools/profiles/update_profiles_v1.py
index d38b3747..128188fb 100644
--- a/reme/memory/vector_tools/profiles/update_profiles_v1.py
+++ b/reme/memory/vector_tools/profiles/update_profiles_v1.py
@@ -1,8 +1,7 @@
-"""Update user profile tool"""
+"""Update user profile tool."""
from loguru import logger
-from .profile_handler import ProfileHandler
from ..base_memory_tool import BaseMemoryTool
from ....core.schema import ToolCall
@@ -113,8 +112,8 @@ class UpdateProfilesV1(BaseMemoryTool):
for target, profile_ids in delete_by_target.items():
if profile_ids:
profile_ids = sorted(set(profile_ids)) # Remove duplicates and sort
- profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=target)
- profile_handler.delete(profile_ids)
+ profile_handler = self.get_profile_handler(target)
+ await profile_handler.adelete(profile_ids)
# Step 2: Prepare all profiles to add (both updated and new)
all_profiles_to_add = []
@@ -158,8 +157,8 @@ class UpdateProfilesV1(BaseMemoryTool):
added_count = len(profiles_to_add)
for target, target_profiles in profiles_by_target.items():
- profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=target)
- new_nodes = profile_handler.add_batch(profiles=target_profiles, ref_memory_id=self.history_id)
+ profile_handler = self.get_profile_handler(target)
+ new_nodes = await profile_handler.aadd_batch(profiles=target_profiles, ref_memory_id=self.history_id)
all_memory_nodes.extend(new_nodes)
# Extend memory_nodes for tracking
diff --git a/reme/reme.py b/reme/reme.py
index f559ad71..c7c04d2e 100644
--- a/reme/reme.py
+++ b/reme/reme.py
@@ -14,6 +14,7 @@ from .memory.vector_tools import (
DelegateTask,
ReadAllProfiles,
ReadHistory,
+ RetrieveProfile,
RetrieveMemory,
UpdateProfilesV1,
)
@@ -55,6 +56,10 @@ class ReMe(Application):
target_task_names: list[str] | None = None,
target_tool_names: list[str] | None = None,
enable_profile: bool = True,
+ profile_backend: str = "filesystem",
+ profile_store_name: str = "profile",
+ profile_collection_name: str | None = None,
+ profile_max_capacity: int = 50,
**kwargs,
):
"""Initialize ReMe with config.
@@ -69,8 +74,37 @@ class ReMe(Application):
```
Args:
+ *args: Positional arguments forwarded to the base `Application`.
+ llm_api_key: API key used by the default LLM backend when provided.
+ llm_base_url: Base URL used by the default LLM backend when provided.
+ embedding_api_key: API key used by the default embedding backend when provided.
+ embedding_base_url: Base URL used by the default embedding backend when provided.
+ working_dir: Directory for generated config, logs, caches, and local stores.
+ config_path: Built-in config name or config file path used to initialize services.
+ enable_logo: Whether to print the ReMe logo during startup.
+ log_to_console: Whether to emit logs to the console.
+ log_to_file: Whether to write logs under `working_dir`.
+ default_llm_config: Overrides for the default LLM configuration.
+ default_embedding_model_config: Overrides for the default embedding model configuration.
+ default_vector_store_config: Configuration for the default memory vector store.
+ Its `collection_name` is used for normal memory storage.
+ default_token_counter_config: Overrides for the default token counter configuration.
+ target_user_names: Personal memory targets to register at initialization.
+ target_task_names: Procedural memory targets to register at initialization.
+ target_tool_names: Tool memory targets to register at initialization.
enable_profile: Whether to enable profile functionality. Set to False when using
- cloud-based vector stores to avoid local file operations. Default is True.
+ profile-free memory flows.
+ profile_backend: Profile storage backend. Use "filesystem" for local JSONL profile
+ files or "vector" for a dedicated profile vector collection.
+ profile_store_name: Internal vector store key used to register and look up the
+ profile vector store in `service_context.vector_stores`. This is not the
+ database collection name.
+ profile_collection_name: Dedicated database collection/table name for vector
+ profiles. When unset, vector profiles use the default memory collection name
+ with a "_profile" suffix.
+ profile_max_capacity: Maximum number of profile rows to keep per memory target.
+ When the limit is exceeded, the oldest profile rows are removed.
+ **kwargs: Additional keyword arguments forwarded to the base `Application`.
"""
super().__init__(
*args,
@@ -92,6 +126,10 @@ class ReMe(Application):
)
self.enable_profile = enable_profile
+ self.profile_backend = profile_backend
+ self.profile_store_name = profile_store_name
+ self.profile_collection_name = profile_collection_name
+ self.profile_max_capacity = profile_max_capacity
memory_target_type_mapping: dict[str, MemoryType] = {}
if target_user_names:
@@ -111,13 +149,16 @@ class ReMe(Application):
self.service_context.memory_target_type_mapping = memory_target_type_mapping
- if self.enable_profile:
+ if self.enable_profile and self.profile_backend == "filesystem":
profile_path = Path(self.service_context.service_config.working_dir) / "profile"
profile_path.mkdir(parents=True, exist_ok=True)
self.profile_dir: str = str(profile_path)
else:
self.profile_dir: str = ""
+ if self.enable_profile and self.profile_backend == "vector":
+ self._ensure_profile_vector_store_config()
+
def _add_meta_memory(self, memory_type: str | MemoryType, memory_target: str):
"""Register or validate a memory target with the given memory type."""
if memory_target in self.service_context.memory_target_type_mapping:
@@ -186,6 +227,38 @@ class ReMe(Application):
return result
return result["answer"]
+ def _ensure_profile_vector_store_config(self) -> None:
+ """Ensure the dedicated profile vector store exists in service config."""
+ vector_store_configs = self.service_context.service_config.vector_stores
+ if "default" not in vector_store_configs:
+ raise RuntimeError("Vector profile backend requires a default vector store configuration")
+
+ default_config = vector_store_configs["default"]
+ profile_collection_name = self.profile_collection_name or f"{default_config.collection_name}_profile"
+
+ if self.profile_store_name in vector_store_configs:
+ if self.profile_collection_name:
+ vector_store_configs[self.profile_store_name] = vector_store_configs[
+ self.profile_store_name
+ ].model_copy(
+ update={"collection_name": profile_collection_name},
+ )
+ return
+
+ vector_store_configs[self.profile_store_name] = default_config.model_copy(
+ update={"collection_name": profile_collection_name},
+ )
+
+ def _get_profile_tool_kwargs(self, raise_exception: bool) -> dict:
+ """Shared profile tool configuration."""
+ return {
+ "profile_dir": self.profile_dir,
+ "profile_backend": self.profile_backend,
+ "profile_store_name": self.profile_store_name,
+ "profile_max_capacity": self.profile_max_capacity,
+ "raise_exception": raise_exception,
+ }
+
async def summarize_memory(
self,
messages: list[Message | dict],
@@ -211,6 +284,7 @@ class ReMe(Application):
format_messages.append(message)
if version == "default":
+ profile_tool_kwargs = self._get_profile_tool_kwargs(raise_exception)
personal_summarizer_tools: list = [
AddDraftAndRetrieveSimilarMemory(
enable_thinking_params=enable_thinking_params,
@@ -229,20 +303,28 @@ class ReMe(Application):
),
]
if self.enable_profile:
+ if self.profile_backend == "vector":
+ profile_context_tool = RetrieveProfile(
+ top_k=min(5, retrieve_top_k),
+ enable_thinking_params=False,
+ enable_memory_target=False,
+ enable_multiple=False,
+ **profile_tool_kwargs,
+ )
+ else:
+ profile_context_tool = ReadAllProfiles(
+ enable_thinking_params=False,
+ enable_memory_target=False,
+ **profile_tool_kwargs,
+ )
personal_summarizer_tools.extend(
[
- ReadAllProfiles(
- enable_thinking_params=False,
- enable_memory_target=False,
- profile_dir=self.profile_dir,
- raise_exception=raise_exception,
- ),
+ profile_context_tool,
UpdateProfilesV1(
enable_thinking_params=enable_thinking_params,
enable_memory_target=False,
enable_multiple=True,
- profile_dir=self.profile_dir,
- raise_exception=raise_exception,
+ **profile_tool_kwargs,
),
],
)
@@ -381,16 +463,24 @@ class ReMe(Application):
self._ensure_started()
if version == "default":
+ profile_tool_kwargs = self._get_profile_tool_kwargs(raise_exception)
personal_retriever_tools = []
if self.enable_profile:
- personal_retriever_tools.append(
- ReadAllProfiles(
+ if self.profile_backend == "vector":
+ profile_context_tool = RetrieveProfile(
+ top_k=min(5, retrieve_top_k),
enable_thinking_params=False,
enable_memory_target=False,
- profile_dir=self.profile_dir,
- raise_exception=raise_exception,
- ),
- )
+ enable_multiple=False,
+ **profile_tool_kwargs,
+ )
+ else:
+ profile_context_tool = ReadAllProfiles(
+ enable_thinking_params=False,
+ enable_memory_target=False,
+ **profile_tool_kwargs,
+ )
+ personal_retriever_tools.append(profile_context_tool)
personal_retriever_tools.extend(
[
RetrieveMemory(
@@ -509,6 +599,34 @@ class ReMe(Application):
return self._unwrap_memory_result(result, "retrieve_memory", return_dict)
+ async def retrieve_profile(
+ self,
+ query: str | list[str],
+ user_name: str,
+ top_k: int = 5,
+ return_dict: bool = False,
+ ) -> str | dict:
+ """Retrieve relevant profile rows for a user."""
+ self._ensure_started()
+ if not self.enable_profile:
+ raise RuntimeError("Profile functionality is disabled.")
+
+ profile_handler = self.get_profile_handler(user_name)
+ if profile_handler is None:
+ raise RuntimeError("Profile functionality is disabled.")
+
+ retrieved_nodes, output = await profile_handler.aretrieve(
+ query=query,
+ limit=top_k,
+ add_profile_id=True,
+ add_history_id=True,
+ )
+ result = {
+ "answer": output or "No matching profiles found.",
+ "retrieved_nodes": retrieved_nodes,
+ }
+ return self._unwrap_memory_result(result, "retrieve_profile", return_dict)
+
async def add_memory(
self,
memory_content: str,
@@ -675,15 +793,23 @@ class ReMe(Application):
@property
def profile_path(self) -> Path | None:
"""Get the path to the profile directory. Returns None if profile is disabled."""
- if not self.enable_profile:
+ if not self.enable_profile or self.profile_backend != "filesystem":
return None
- return Path(self.profile_dir) / self.default_vector_store.collection_name
+ collection_name = self.service_context.service_config.vector_stores["default"].collection_name
+ return Path(self.profile_dir) / collection_name
def get_profile_handler(self, user_name: str) -> ProfileHandler | None:
"""Get the profile handler for the specified user. Returns None if profile is disabled."""
if not self.enable_profile:
return None
- return ProfileHandler(memory_target=user_name, profile_path=self.profile_path)
+ return ProfileHandler(
+ memory_target=user_name,
+ profile_path=self.profile_path,
+ service_context=self.service_context,
+ profile_backend=self.profile_backend,
+ profile_store_name=self.profile_store_name,
+ max_capacity=self.profile_max_capacity,
+ )
def main():
diff --git a/tests/test_reme_memory_error_handling.py b/tests/test_reme_memory_error_handling.py
index 99953efd..563e8221 100644
--- a/tests/test_reme_memory_error_handling.py
+++ b/tests/test_reme_memory_error_handling.py
@@ -1,9 +1,14 @@
"""Tests for ReMe memory error handling and raise_exception propagation."""
+from types import SimpleNamespace
+
import pytest
import reme.reme as reme_module
-from reme import ReMe
+from reme.core.runtime_context import RuntimeContext
+from reme.core.schema import MemoryNode
+from reme.memory.vector_tools.history.read_history import ReadHistory
+from reme.reme import ReMe
class Recorder:
@@ -130,3 +135,34 @@ async def test_summarize_memory_raises_runtime_error_for_unstructured_result(mon
messages=[{"role": "user", "content": "hi", "time_created": "2026-03-20 10:00:00"}],
task_name="demo-task",
)
+
+
+@pytest.mark.asyncio
+async def test_read_history_accepts_single_history_id_in_multiple_mode():
+ """Verify multiple-mode history lookup accepts a single history_id string."""
+
+ class FakeVectorStore:
+ """Minimal vector store stub for ReadHistory tests."""
+
+ async def get(self, vector_ids):
+ """Return the requested history node."""
+ assert vector_ids == ["history_123"]
+ node = MemoryNode(
+ memory_id="history_123",
+ memory_type="history",
+ memory_target="alice",
+ content="Alice said hello.",
+ )
+ return [node.to_vector_node()]
+
+ tool = ReadHistory(enable_multiple=True)
+ tool._vector_store = FakeVectorStore() # pylint: disable=protected-access
+ tool.context = RuntimeContext(
+ history_id="history_123",
+ retrieved_nodes=[],
+ service_context=SimpleNamespace(memory_target_type_mapping={"alice": "personal"}),
+ )
+
+ result = await tool.execute()
+
+ assert "Historical Dialogue[history_123]" in result
From 72eabfa858bd331d021310435f77bb718931ad47 Mon Sep 17 00:00:00 2001
From: Zhouwk <57825291+nitwtog@users.noreply.github.com>
Date: Thu, 30 Apr 2026 10:19:36 +0800
Subject: [PATCH 04/16] fix(user profile): update locomo benchmark and update
vector based profile code (#225)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* feat(reme): 添加配置选项以启用或禁用个人资料功能
- 在 ReMe 初始化方法中添加 enable_profile 参数，默认值为 True
- 根据 enable_profile 设置决定是否创建 profile 目录和设置 profile_dir
- 在 PersonalSummarizer 中根据 enable_profile 条件性地添加个人资料相关工具
- 在 PersonalRetriever 中根据 enable_profile 条件性地添加 ReadAllProfiles 工具
- 修改 profile_path 属性以在禁用个人资料时返回 None
- 修改 get_profile_handler 方法以在禁用个人资料时返回 None
- 为 enable_profile 参数添加文档说明其用于云向量存储场景
* refactor(benchmark): 重构LongMemEval基准测试中的ReMe实例管理
- 移除未使用的shutil导入
- 将固定的ReMe实例改为每个问题创建独立实例以实现隔离
- 更新LLM配置名称从qwen3-max-think到qwen-max-t
- 修改模型调用逻辑使用正确的model_name参数
- 添加qwen-flash和GPT-4o-mini等新模型配置
- 统一使用"User"作为用户名，通过集合名实现隔离
- 调整并发处理数从4降至1，批处理大小从10增至30
- 每个问题类型采样数从2增至4
- 添加异步上下文管理确保资源正确释放
* reformat 2 files
* refactor(benchmark): 重构长记忆评估中的模型配置
- 将原有的 eval_model_name 替换为专门的 retrieve_model_name 用于检索操作
- 添加对 qwen-max 模型配置的支持
- 更新参数解析器以支持新的检索模型参数
- 修改最大并发数默认值从 1 提升到 4
- 调整样本数量默认值从 4 减少到 1
- 统一模型参数命名规范，区分摘要、检索和评估模型
- 优化内存处理器初始化逻辑，支持独立的检索模型配置
* fix(benchmark): 移除数据路径默认值并设为必填参数
- 将LongMemEval评估脚本中的data_path参数改为必需参数
- 将HaluMem评估脚本中的data_path参数改为必需参数
- 删除了硬编码的默认文件路径配置
- 强制用户显式指定数据集文件路径以避免路径错误
* Update __init__.py
* Update __init__.py
* fix(benchmark): 修复ReMe评估中的模型配置和空值处理问题
- 移除了retrieve_memory调用中不需要的llm_config_name参数
- 修复了长字符串打印的换行格式问题
- 添加了eval_result为空时的初始化处理
- 在accuracy评估中加入了eval_model_name参数传递
* style(benchmark): 格式化模型名称打印输出
- 移除了多行字符串中的换行符和多余空格
- 将模型名称信息合并为单行连续显示
- 保持了原有的打印格式和信息完整性
* docs(readme): 更新文档添加实验结果表格
- 在英文版 README 中添加 🧪 Experiments 章节
- 添加 LoCoMo 和 HaluMem 两个基准测试的结果表格
- 在中文版 README_ZH 中添加 🧪 实验 章节
- 添加 LoCoMo 和 HaluMem 测试集的实验配置说明
- 添加完整的实验数据对比表格和评估协议说明
* docs(readme): 更新文档中的内存系统链接
- 为基于文件的记忆系统添加锚点链接
- 为基于向量库的记忆系统添加锚点链接
- 修复英文文档中的链接格式
- 修复中文文档中的链接格式和空行问题
* docs(readme): update experimental results section in documentation
- Remove outdated experimental data placeholder "Coming soon..."
- Add complete evaluation results for LoCoMo and HaluMem benchmarks
- Include detailed performance metrics tables for all memory methods
- Update experimental settings description with ReMe backbone details
- Align evaluation protocol information with LLM-as-a-Judge approach
- Maintain consistent formatting between English and Chinese documentation
* docs(benchmark): add quick start guides for halumem and longmemeval experiments
- Created HaluMem experiment quick start guide with ReMe integration setup
- Added detailed steps for installing ReMe environment using conda
- Included repository cloning instructions for HaluMem benchmark
- Provided complete command examples for running HaluMem experiments
- Created LongMeMEval quick start guide with data download procedures
- Added wget commands for downloading cleaned dataset files
- Included evaluation script instructions for computing experiment statistics
- Documented parameter configurations for different model types and batch sizes
* docs(longmemeval): update quickstart guide documentation
- Changed project name from Halumem to Longmemeval in title
- Updated description to reference Longmemeval experiments instead of Halumem
- Maintained existing ReMe integration instructions unchanged
* chore(logger): add test comment to logger configuration
- Added test comment in logger utility function
- Removed duplicate log handling by keeping the remove() call
* chore(logger): add test comment to logger configuration
- Added test comment in logger utility function
- Removed duplicate log handling by keeping the remove() call
* feat(core): add file logging capability to application
- Added log_to_file parameter to Application class constructor
- Integrated log_to_file option in logger initialization
- Updated ServiceContext to support file logging configuration
- Modified init_logger function to conditionally enable file logging
- Added log_to_file field to ServiceConfig schema
- Updated ReMe class to include file logging option
- Wrapped file logging setup in conditional check to prevent unnecessary operations
* docs(benchmark): update HaluMem quickstart guide with dataset download instructions
- Replace repository cloning with direct dataset download using curl
- Add commands to download HaluMem-Medium.jsonl and HaluMem-Long.jsonl files
- Include both official Hugging Face and mirror download sources
- Update data path reference from nested directory to local data folder
- Add dataset page link and mirror usage instructions for mainland China access
* feat(memory): add profile retrieval tool and refactor profile management
- Introduce RetrieveProfile tool for fetching specific user profiles
- Refactor ProfileHandler to support both filesystem and vector backends
- Add async methods to ProfileHandler with synchronous fallbacks
- Update PersonalRetriever to support two-stage profile and memory retrieval
- Enhance PersonalSummarizer with improved tool partitioning logic
- Add profile_backend, profile_store_name, and profile_max_capacity configuration options
- Replace direct ProfileHandler imports with get_profile_handler method
- Implement profile search functionality with dedicated prompts and workflows
- Add FileProfileBackend and VectorProfileBackend implementations
- Update base memory tool with new profile configuration parameters
* feat(profile): add custom profile collection name support
- Add profile_collection_name parameter to Application constructor
- Allow custom database collection name for vector profiles instead of default suffix
- Update profile vector store configuration logic to use custom collection name
- Modify _ensure_profile_vector_store_config to handle custom collection names
- Update docstring with detailed parameter descriptions for profile configuration options
* test(history): add single history id acceptance test for multiple mode
- Add test case to verify multiple-mode history lookup accepts a single history_id string
- Create FakeVectorStore stub with minimal implementation for ReadHistory tests
- Return requested history node from vector store mock
- Initialize ReadHistory tool with multiple mode enabled
- Add pylint disable comment for protected access to vector store property
* refactor(memory): update profile handler and vector tools with improved formatting and error handling
- Add module docstring to profiles/__init__.py
- Add pylint disable comments for no-name-in-module and missing-function-docstring
- Format long error message in ProfileHandler.sync_run method for better readability
- Reformat parameters in ProfileHandler.aadd method to separate lines
- Update model_copy call in reme.py to span multiple lines for better readability
- Format aadd_batch call in update_profile.py to span multiple lines
* feat(profiles): add profile management system with file and vector storage backends
- Add FileProfileBackend for filesystem-based profile persistence
- Add VectorProfileBackend for vector store-based profile management
- Create abstract BaseProfileBackend interface for profile operations
- Implement ProfileVectorHandler for vector-backed profile storage
- Add RetrieveProfile tool for semantic profile retrieval
- Update eval_reme.py to use user_message_s2 for retriever prompt
- Modify eval_reme.yaml to use {profiles} instead of {user_profile}
- Implement complete CRUD operations for profile management
- Add batch operations for efficient profile handling
- Include search functionality with semantic matching capabilities
- Add capacity limits and automatic cleanup for profile storage
* docs(profiles): add comprehensive docstrings for profile backend and handler methods
- Added documentation for get_all_sync, get_by_sync, delete_sync, delete_all_sync methods
- Documented add_sync and add_batch_sync functionality with deduping behavior
- Added docstrings for update_sync and search_sync operations
- Updated ProfileHandler.format_node method with proper documentation
- Refactored private _format_node to public format_node method
- Added comprehensive documentation for profile vector handler operations
- Documented _vector_profile_matches, _get_by_profile_id, _get_by_profile_key helper methods
- Added docstrings for retrieve_profile functionality and formatting methods
---
benchmark/locomo/eval_reme.py | 2 +-
benchmark/locomo/eval_reme.yaml | 2 +-
.../profiles/file_profile_backend.py | 234 +++++++++++++++++
.../vector_tools/profiles/profile_backend.py | 51 ++++
.../vector_tools/profiles/profile_handler.py | 7 +-
.../profiles/profile_vector_handler.py | 245 ++++++++++++++++++
.../vector_tools/profiles/retrieve_profile.py | 102 ++++++++
.../profiles/vector_profile_backend.py | 55 ++++
8 files changed, 693 insertions(+), 5 deletions(-)
create mode 100644 reme/memory/vector_tools/profiles/file_profile_backend.py
create mode 100644 reme/memory/vector_tools/profiles/profile_backend.py
create mode 100644 reme/memory/vector_tools/profiles/profile_vector_handler.py
create mode 100644 reme/memory/vector_tools/profiles/retrieve_profile.py
create mode 100644 reme/memory/vector_tools/profiles/vector_profile_backend.py
diff --git a/benchmark/locomo/eval_reme.py b/benchmark/locomo/eval_reme.py
index 82d91031..bb424aee 100644
--- a/benchmark/locomo/eval_reme.py
+++ b/benchmark/locomo/eval_reme.py
@@ -643,7 +643,7 @@ class LocomoEvaluator:
},
"personal_retriever": {
"prompt_dict": {
- "user_message": self.retriever_prompt,
+ "user_message_s2": self.retriever_prompt,
},
"params": {
"return_memory_nodes": True,
diff --git a/benchmark/locomo/eval_reme.yaml b/benchmark/locomo/eval_reme.yaml
index 113431b3..5228884a 100644
--- a/benchmark/locomo/eval_reme.yaml
+++ b/benchmark/locomo/eval_reme.yaml
@@ -132,7 +132,7 @@ user_message_retrieve: |
You are a Memory Retrieval Agent specialized in retrieving {memory_type} memories about {memory_target}.
## User Profile
- {user_profile}
+ {profiles}
## User Question
{context}
diff --git a/reme/memory/vector_tools/profiles/file_profile_backend.py b/reme/memory/vector_tools/profiles/file_profile_backend.py
new file mode 100644
index 00000000..e6226459
--- /dev/null
+++ b/reme/memory/vector_tools/profiles/file_profile_backend.py
@@ -0,0 +1,234 @@
+"""Filesystem-backed profile storage."""
+
+from pathlib import Path
+
+from loguru import logger
+
+from .profile_backend import BaseProfileBackend
+from ....core.enumeration import MemoryType
+from ....core.schema import MemoryNode
+from ....core.utils import CacheHandler, deduplicate_memories
+
+
+class FileProfileBackend(BaseProfileBackend):
+ """Persist user profiles in local JSONL cache files."""
+
+ def __init__(self, profile_path: str | Path, memory_target: str, max_capacity: int = 50):
+ super().__init__(memory_target=memory_target, max_capacity=max_capacity)
+ self.cache_key: str = self.memory_target.replace(" ", "_").lower()
+ self.cache_handler: CacheHandler = CacheHandler(profile_path)
+
+ def _load_nodes(self) -> list[MemoryNode]:
+ cached_data = self.cache_handler.load(self.cache_key, auto_clean=False)
+ if not cached_data:
+ return []
+ return [MemoryNode(**data) for data in cached_data]
+
+ def _save_nodes(self, nodes: list[MemoryNode], apply_limits: bool = True):
+ if apply_limits:
+ nodes = deduplicate_memories(nodes)
+
+ if len(nodes) > self.max_capacity:
+ sorted_nodes = sorted(nodes, key=lambda n: n.message_time)
+ removed_count = len(sorted_nodes) - self.max_capacity
+ nodes = sorted_nodes[removed_count:]
+ logger.info(
+ f"Capacity limit reached: removed {removed_count} oldest profiles "
+ f"(kept {len(nodes)}/{self.max_capacity})",
+ )
+
+ nodes_data = [node.model_dump(exclude_none=True) for node in nodes]
+ self.cache_handler.save(self.cache_key, nodes_data)
+ logger.info(f"Saved {len(nodes)} profiles to {self.cache_key}")
+
+ def get_all_sync(self) -> list[MemoryNode]:
+ """Load all profile nodes from cache, ordered by ``message_time``."""
+ nodes = self._load_nodes()
+ nodes.sort(key=lambda n: n.message_time)
+ return nodes
+
+ def get_by_sync(self, *, profile_id: str | None = None, profile_key: str | None = None) -> MemoryNode | None:
+ """Return the first node matching ``profile_id`` or ``profile_key``."""
+ if not profile_id and not profile_key:
+ raise ValueError("Must provide either profile_id or profile_key")
+
+ for node in self._load_nodes():
+ if profile_id and node.memory_id == profile_id:
+ return node
+ if profile_key and node.when_to_use == profile_key:
+ return node
+ return None
+
+ def delete_sync(self, profile_id: str | list[str]) -> bool | int:
+ """Remove one id, many ids, or none; returns bool, count, or 0/false if nothing removed."""
+ nodes = self._load_nodes()
+ original_count = len(nodes)
+
+ if isinstance(profile_id, list):
+ profile_ids_set = set(profile_id)
+ nodes = [n for n in nodes if n.memory_id not in profile_ids_set]
+ deleted_count = original_count - len(nodes)
+ if deleted_count == 0:
+ logger.warning(f"No profiles found to delete from {len(profile_id)} IDs")
+ return 0
+
+ self._save_nodes(nodes, apply_limits=False)
+ logger.info(f"Batch deleted {deleted_count} profiles")
+ return deleted_count
+
+ nodes = [n for n in nodes if n.memory_id != profile_id]
+ if len(nodes) == original_count:
+ logger.warning(f"Profile {profile_id} not found")
+ return False
+
+ self._save_nodes(nodes, apply_limits=False)
+ logger.info(f"Deleted profile {profile_id}")
+ return True
+
+ def delete_all_sync(self) -> int:
+ """Clear every cached profile for this target; returns how many were stored."""
+ nodes = self._load_nodes()
+ count = len(nodes)
+ self._save_nodes([], apply_limits=False)
+ logger.info(f"Deleted all {count} profiles")
+ return count
+
+ def add_sync(self, message_time: str, profile_key: str, profile_value: str, ref_memory_id: str = "") -> MemoryNode:
+ """Append a profile row, replacing any existing row with the same key."""
+ nodes = self._load_nodes()
+
+ new_node = MemoryNode(
+ memory_type=MemoryType.PERSONAL,
+ memory_target=self.memory_target,
+ when_to_use=profile_key,
+ content=profile_value,
+ message_time=message_time,
+ ref_memory_id=ref_memory_id,
+ )
+
+ original_count = len(nodes)
+ nodes = [n for n in nodes if n.when_to_use != profile_key]
+ if len(nodes) < original_count:
+ logger.info(f"Removed {original_count - len(nodes)} duplicate profile(s) with key: {profile_key}")
+
+ nodes.append(new_node)
+ self._save_nodes(nodes)
+ logger.info(f"Added profile: {profile_key}={profile_value}")
+ return new_node
+
+ def add_batch_sync(self, profiles: list[dict], ref_memory_id: str = "") -> list[MemoryNode]:
+ """Insert many profiles in one write, deduping by key against existing rows."""
+ if not profiles:
+ return []
+
+ nodes = self._load_nodes()
+ new_nodes = [
+ MemoryNode(
+ memory_type=MemoryType.PERSONAL,
+ memory_target=self.memory_target,
+ when_to_use=p.get("profile_key", ""),
+ content=p.get("profile_value", ""),
+ message_time=p.get("message_time", ""),
+ ref_memory_id=ref_memory_id,
+ )
+ for p in profiles
+ ]
+
+ new_keys = {n.when_to_use for n in new_nodes}
+ original_count = len(nodes)
+ nodes = [n for n in nodes if n.when_to_use not in new_keys]
+ if len(nodes) < original_count:
+ logger.info(f"Removed {original_count - len(nodes)} duplicate profile(s) with matching keys")
+
+ nodes.extend(new_nodes)
+ self._save_nodes(nodes)
+ logger.info(f"Batch added {len(new_nodes)} profiles")
+ return new_nodes
+
+ def update_sync(
+ self,
+ profile_id: str,
+ message_time: str,
+ profile_key: str,
+ profile_value: str,
+ ) -> MemoryNode | None:
+ """Update fields for ``profile_id``; return ``None`` if that id is missing."""
+ nodes = self._load_nodes()
+ target_node = None
+ for node in nodes:
+ if node.memory_id == profile_id:
+ node.when_to_use = profile_key
+ node.content = profile_value
+ node.message_time = message_time
+ target_node = node
+ break
+
+ if target_node is None:
+ logger.warning(f"Profile {profile_id} not found")
+ return None
+
+ self._save_nodes(nodes, apply_limits=False)
+ logger.info(f"Updated profile {profile_id}: {profile_key}={profile_value}")
+ return target_node
+
+ def search_sync(self, query: str | list[str], limit: int = 5) -> list[MemoryNode]:
+ """Simple substring/token match over key and content, best matches first."""
+ queries = [query] if isinstance(query, str) else query
+ query_terms = [q.strip().lower() for q in queries if q and q.strip()]
+ if not query_terms:
+ return []
+
+ scored_nodes = []
+ for node in self.get_all_sync():
+ profile_key = str(node.metadata.get("profile_key", node.when_to_use)).lower()
+ haystack = f"{profile_key}: {node.content}".lower()
+ score = 0
+ for term in query_terms:
+ if term in haystack:
+ score += len(term) + 10
+ else:
+ token_hits = sum(1 for token in term.split() if token and token in haystack)
+ score += token_hits
+
+ if score > 0:
+ node.score = float(score)
+ scored_nodes.append(node)
+
+ scored_nodes.sort(key=lambda n: (n.score, n.message_time), reverse=True)
+ return scored_nodes[:limit]
+
+ async def get_all(self) -> list[MemoryNode]:
+ return self.get_all_sync()
+
+ async def get_by(self, *, profile_id: str | None = None, profile_key: str | None = None) -> MemoryNode | None:
+ return self.get_by_sync(profile_id=profile_id, profile_key=profile_key)
+
+ async def delete(self, profile_id: str | list[str]) -> bool | int:
+ return self.delete_sync(profile_id)
+
+ async def delete_all(self) -> int:
+ return self.delete_all_sync()
+
+ async def add(
+ self,
+ message_time: str,
+ profile_key: str,
+ profile_value: str,
+ ref_memory_id: str = "",
+ ) -> MemoryNode:
+ return self.add_sync(message_time, profile_key, profile_value, ref_memory_id)
+
+ async def add_batch(self, profiles: list[dict], ref_memory_id: str = "") -> list[MemoryNode]:
+ return self.add_batch_sync(profiles, ref_memory_id)
+
+ async def update(
+ self,
+ profile_id: str,
+ message_time: str,
+ profile_key: str,
+ profile_value: str,
+ ) -> MemoryNode | None:
+ return self.update_sync(profile_id, message_time, profile_key, profile_value)
+
+ async def search(self, query: str | list[str], limit: int = 5) -> list[MemoryNode]:
+ return self.search_sync(query, limit)
diff --git a/reme/memory/vector_tools/profiles/profile_backend.py b/reme/memory/vector_tools/profiles/profile_backend.py
new file mode 100644
index 00000000..8ffcb9bc
--- /dev/null
+++ b/reme/memory/vector_tools/profiles/profile_backend.py
@@ -0,0 +1,51 @@
+"""Profile backend abstractions."""
+
+from abc import ABC, abstractmethod
+
+from ....core.schema import MemoryNode
+
+
+class BaseProfileBackend(ABC):
+ """Abstract interface for profile storage backends."""
+
+ def __init__(self, memory_target: str, max_capacity: int = 50):
+ self.memory_target = memory_target
+ self.max_capacity = max_capacity
+
+ @abstractmethod
+ async def get_all(self) -> list[MemoryNode]:
+ """Return all profile rows for the current user."""
+
+ @abstractmethod
+ async def get_by(self, *, profile_id: str | None = None, profile_key: str | None = None) -> MemoryNode | None:
+ """Return one profile row by id or key."""
+
+ @abstractmethod
+ async def delete(self, profile_id: str | list[str]) -> bool | int:
+ """Delete one or more profile rows."""
+
+ @abstractmethod
+ async def delete_all(self) -> int:
+ """Delete all profile rows for the current user."""
+
+ @abstractmethod
+ async def add(self, message_time: str, profile_key: str, profile_value: str, ref_memory_id: str = "") -> MemoryNode:
+ """Add a single profile row."""
+
+ @abstractmethod
+ async def add_batch(self, profiles: list[dict], ref_memory_id: str = "") -> list[MemoryNode]:
+ """Add multiple profile rows."""
+
+ @abstractmethod
+ async def update(
+ self,
+ profile_id: str,
+ message_time: str,
+ profile_key: str,
+ profile_value: str,
+ ) -> MemoryNode | None:
+ """Update one profile row."""
+
+ @abstractmethod
+ async def search(self, query: str | list[str], limit: int = 5) -> list[MemoryNode]:
+ """Search profile rows relevant to the query."""
diff --git a/reme/memory/vector_tools/profiles/profile_handler.py b/reme/memory/vector_tools/profiles/profile_handler.py
index 0df3707a..145caa6e 100644
--- a/reme/memory/vector_tools/profiles/profile_handler.py
+++ b/reme/memory/vector_tools/profiles/profile_handler.py
@@ -115,7 +115,8 @@ class ProfileHandler:
return await self.backend.search(query=query, limit=limit)
@staticmethod
- def _format_node(node: MemoryNode, add_profile_id: bool = False, add_history_id: bool = False) -> str:
+ def format_node(node: MemoryNode, add_profile_id: bool = False, add_history_id: bool = False) -> str:
+ """Render a profile ``MemoryNode`` as a single-line string for tools/logs."""
parts = []
profile_key = str(node.metadata.get("profile_key", node.when_to_use))
@@ -134,7 +135,7 @@ class ProfileHandler:
async def aread_all(self, add_profile_id: bool = False, add_history_id: bool = False) -> str:
nodes = await self.aget_all()
- formatted_profiles = [self._format_node(node, add_profile_id, add_history_id) for node in nodes]
+ formatted_profiles = [self.format_node(node, add_profile_id, add_history_id) for node in nodes]
logger.info(f"Read {len(formatted_profiles)} profiles from {self.cache_key}")
return "\n".join(formatted_profiles).strip()
@@ -146,7 +147,7 @@ class ProfileHandler:
add_history_id: bool = False,
) -> tuple[list[MemoryNode], str]:
nodes = await self.asearch(query=query, limit=limit)
- formatted_profiles = [self._format_node(node, add_profile_id, add_history_id) for node in nodes]
+ formatted_profiles = [self.format_node(node, add_profile_id, add_history_id) for node in nodes]
return nodes, "\n".join(formatted_profiles).strip()
def delete(self, profile_id: str | list[str]) -> bool | int:
diff --git a/reme/memory/vector_tools/profiles/profile_vector_handler.py b/reme/memory/vector_tools/profiles/profile_vector_handler.py
new file mode 100644
index 00000000..f03e6a96
--- /dev/null
+++ b/reme/memory/vector_tools/profiles/profile_vector_handler.py
@@ -0,0 +1,245 @@
+"""Vector-backed handler for bounded user profiles."""
+
+import hashlib
+
+from loguru import logger
+
+from ....core import ServiceContext
+from ....core.enumeration import MemoryType
+from ....core.schema import MemoryNode
+from ....core.vector_store import BaseVectorStore
+
+
+class ProfileVectorHandler:
+ """Manage profile rows stored in a dedicated vector collection."""
+
+ PROFILE_KIND = "profile"
+
+ def __init__(
+ self,
+ memory_target: str,
+ service_context: ServiceContext,
+ vector_store_name: str = "profile",
+ max_capacity: int = 50,
+ ):
+ self.memory_target = memory_target
+ self.service_context = service_context
+ self.vector_store_name = vector_store_name
+ self.max_capacity = max_capacity
+ self.vector_store: BaseVectorStore = service_context.vector_stores[vector_store_name]
+
+ @staticmethod
+ def build_retrieval_text(profile_key: str, profile_value: str) -> str:
+ """Build the text that will be embedded for semantic profile retrieval."""
+ return f"{profile_key}: {profile_value}".strip(": ")
+
+ def build_profile_id(self, profile_key: str) -> str:
+ """Build a stable id from user and key."""
+ hash_obj = hashlib.sha256(f"{self.memory_target}\n{profile_key}".encode("utf-8"))
+ return hash_obj.hexdigest()[:16]
+
+ def _base_filters(self) -> dict:
+ """Filters shared by all profile rows in the vector collection."""
+ return {
+ "memory_type": MemoryType.IDENTITY.value,
+ "memory_target": self.memory_target,
+ "profile_kind": self.PROFILE_KIND,
+ }
+
+ def _build_profile_node(self, profile: dict, ref_memory_id: str = "") -> MemoryNode:
+ """Turn a profile dict into a ``MemoryNode`` for upsert into the vector store."""
+ profile_key = profile.get("profile_key", "").strip()
+ profile_value = profile.get("profile_value", "").strip()
+ message_time = profile.get("message_time", "")
+ ref_id = profile.get("ref_memory_id", ref_memory_id)
+ metadata = dict(profile.get("metadata", {}))
+ metadata.update(
+ {
+ "profile_key": profile_key,
+ "profile_kind": self.PROFILE_KIND,
+ "profile_backend": "vector",
+ },
+ )
+ return MemoryNode(
+ memory_id=self.build_profile_id(profile_key),
+ memory_type=MemoryType.IDENTITY,
+ memory_target=self.memory_target,
+ when_to_use=self.build_retrieval_text(profile_key, profile_value),
+ content=profile_value,
+ message_time=message_time,
+ ref_memory_id=ref_id,
+ metadata=metadata,
+ )
+
+ def _vector_profile_matches(self, memory_node: MemoryNode) -> bool:
+ """True if ``memory_node`` belongs to this handler's target and profile kind."""
+ if memory_node.memory_target != self.memory_target:
+ return False
+ if memory_node.memory_type is not MemoryType.IDENTITY:
+ return False
+ if memory_node.metadata.get("profile_kind") != self.PROFILE_KIND:
+ return False
+ return True
+
+ async def _get_by_profile_id(self, profile_id: str) -> MemoryNode | None:
+ """Load by vector id and validate filters."""
+ try:
+ vector_node = await self.vector_store.get(profile_id)
+ except KeyError:
+ logger.warning(f"Profile {profile_id} not found in vector store")
+ return None
+ if vector_node is None:
+ logger.warning(f"Profile {profile_id} not found in vector store")
+ return None
+ memory_node = MemoryNode.from_vector_node(vector_node)
+ if not self._vector_profile_matches(memory_node):
+ return None
+ return memory_node
+
+ async def _get_by_profile_key(self, profile_key: str) -> MemoryNode | None:
+ """Load the single row matching ``profile_key`` under base filters."""
+ filters = {**self._base_filters(), "profile_key": profile_key}
+ vector_nodes = await self.vector_store.list(filters=filters, limit=1)
+ if not vector_nodes:
+ return None
+ return MemoryNode.from_vector_node(vector_nodes[0])
+
+ async def get_all(self) -> list[MemoryNode]:
+ """List every profile row for this memory target, sorted by store."""
+ vector_nodes = await self.vector_store.list(
+ filters=self._base_filters(),
+ sort_key="message_time",
+ reverse=False,
+ )
+ return [MemoryNode.from_vector_node(node) for node in vector_nodes]
+
+ async def get_by(self, *, profile_id: str | None = None, profile_key: str | None = None) -> MemoryNode | None:
+ """Return one profile by stable id or by logical profile key."""
+ if not profile_id and not profile_key:
+ raise ValueError("Must provide either profile_id or profile_key")
+ if profile_id:
+ return await self._get_by_profile_id(profile_id)
+ return await self._get_by_profile_key(profile_key or "")
+
+ async def delete(self, profile_id: str | list[str]) -> bool | int:
+ """Delete one id, many ids, or report zero/false when nothing matched."""
+ if isinstance(profile_id, list):
+ profile_ids = list(dict.fromkeys(pid for pid in profile_id if pid))
+ if not profile_ids:
+ return 0
+ existing_nodes = []
+ for pid in profile_ids:
+ node = await self.get_by(profile_id=pid)
+ if node is not None:
+ existing_nodes.append(node)
+ if not existing_nodes:
+ return 0
+ await self.vector_store.delete([node.memory_id for node in existing_nodes])
+ return len(existing_nodes)
+
+ existing_node = await self.get_by(profile_id=profile_id)
+ if existing_node is None:
+ return False
+ await self.vector_store.delete(existing_node.memory_id)
+ return True
+
+ async def delete_all(self) -> int:
+ """Remove all profile vectors for this target; returns how many were deleted."""
+ nodes = await self.get_all()
+ if not nodes:
+ return 0
+ await self.vector_store.delete([node.memory_id for node in nodes])
+ return len(nodes)
+
+ async def add_batch(self, profiles: list[dict], ref_memory_id: str = "") -> list[MemoryNode]:
+ """Upsert many profiles at once (last dict wins per key), then enforce capacity."""
+ if not profiles:
+ return []
+
+ deduped_profiles: dict[str, dict] = {}
+ for profile in profiles:
+ profile_key = profile.get("profile_key", "").strip()
+ if not profile_key:
+ continue
+ deduped_profiles[profile_key] = profile
+
+ new_nodes = [
+ self._build_profile_node(profile, ref_memory_id=ref_memory_id) for profile in deduped_profiles.values()
+ ]
+ if not new_nodes:
+ return []
+
+ await self.vector_store.delete([node.memory_id for node in new_nodes])
+ await self.vector_store.insert([node.to_vector_node() for node in new_nodes])
+ await self.enforce_capacity()
+ return new_nodes
+
+ async def add(self, message_time: str, profile_key: str, profile_value: str, ref_memory_id: str = "") -> MemoryNode:
+ """Insert or replace a single profile row."""
+ nodes = await self.add_batch(
+ [
+ {
+ "message_time": message_time,
+ "profile_key": profile_key,
+ "profile_value": profile_value,
+ },
+ ],
+ ref_memory_id=ref_memory_id,
+ )
+ return nodes[0]
+
+ async def update(
+ self,
+ profile_id: str,
+ message_time: str,
+ profile_key: str,
+ profile_value: str,
+ ) -> MemoryNode | None:
+ """Replace content and key for ``profile_id``; return ``None`` if missing."""
+ existing_node = await self.get_by(profile_id=profile_id)
+ if existing_node is None:
+ return None
+
+ new_node = self._build_profile_node(
+ {
+ "message_time": message_time,
+ "profile_key": profile_key,
+ "profile_value": profile_value,
+ "ref_memory_id": existing_node.ref_memory_id,
+ "metadata": existing_node.metadata,
+ },
+ )
+
+ if existing_node.memory_id != new_node.memory_id:
+ await self.vector_store.delete(existing_node.memory_id)
+ else:
+ await self.vector_store.delete(new_node.memory_id)
+
+ await self.vector_store.insert(new_node.to_vector_node())
+ await self.enforce_capacity()
+ return new_node
+
+ async def search(self, query: str | list[str], limit: int = 5) -> list[MemoryNode]:
+ """Semantic search with de-duplication across multiple query strings."""
+ queries = [query] if isinstance(query, str) else query
+ seen_nodes: dict[str, MemoryNode] = {}
+ for item in queries:
+ if not item or not item.strip():
+ continue
+ vector_nodes = await self.vector_store.search(item, limit=limit, filters=self._base_filters())
+ for vector_node in vector_nodes:
+ memory_node = MemoryNode.from_vector_node(vector_node)
+ seen_nodes[memory_node.memory_id] = memory_node
+ nodes = list(seen_nodes.values())
+ nodes.sort(key=lambda node: (node.score, node.message_time), reverse=True)
+ return nodes[:limit]
+
+ async def enforce_capacity(self):
+ """Drop oldest rows when count exceeds ``max_capacity``."""
+ nodes = await self.get_all()
+ overflow = len(nodes) - self.max_capacity
+ if overflow <= 0:
+ return
+
+ to_delete = [node.memory_id for node in nodes[:overflow]]
+ await self.vector_store.delete(to_delete)
diff --git a/reme/memory/vector_tools/profiles/retrieve_profile.py b/reme/memory/vector_tools/profiles/retrieve_profile.py
new file mode 100644
index 00000000..38337bd8
--- /dev/null
+++ b/reme/memory/vector_tools/profiles/retrieve_profile.py
@@ -0,0 +1,102 @@
+"""Retrieve relevant profile rows."""
+
+from loguru import logger
+
+from .profile_handler import ProfileHandler
+from ..base_memory_tool import BaseMemoryTool
+from ....core.schema import MemoryNode, ToolCall
+
+
+class RetrieveProfile(BaseMemoryTool):
+ """Tool to retrieve relevant profiles using the configured backend."""
+
+ def __init__(self, top_k: int = 5, enable_memory_target: bool = False, **kwargs):
+ super().__init__(**kwargs)
+ self.top_k = top_k
+ self.enable_memory_target = enable_memory_target
+
+ def _build_query_parameters(self) -> dict:
+ properties = {
+ "query": {
+ "type": "string",
+ "description": "query",
+ },
+ }
+ required = ["query"]
+ if self.enable_memory_target:
+ properties["memory_target"] = {
+ "type": "string",
+ "description": "memory_target",
+ }
+ required.append("memory_target")
+ return {
+ "type": "object",
+ "properties": properties,
+ "required": required,
+ }
+
+ def _build_tool_call(self) -> ToolCall:
+ return ToolCall(
+ **{
+ "description": "Retrieve relevant user profiles using semantic matching.",
+ "parameters": self._build_query_parameters(),
+ },
+ )
+
+ def _build_multiple_tool_call(self) -> ToolCall:
+ return ToolCall(
+ **{
+ "description": "Retrieve relevant user profiles using semantic matching.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "query_items": {
+ "type": "array",
+ "description": "List of query items.",
+ "items": self._build_query_parameters(),
+ },
+ },
+ "required": ["query_items"],
+ },
+ },
+ )
+
+ async def execute(self):
+ if self.enable_multiple:
+ query_items = self.context.get("query_items", [])
+ else:
+ query_items = [self.context]
+
+ queries_by_target: dict[str, list[str]] = {}
+ for item in query_items:
+ target = item["memory_target"] if self.enable_memory_target else self.memory_target
+ queries_by_target.setdefault(target, []).append(item["query"])
+
+ profile_nodes: list[MemoryNode] = []
+ for target, queries in queries_by_target.items():
+ profile_handler = self.get_profile_handler(target)
+ nodes, _ = await profile_handler.aretrieve(
+ query=queries,
+ limit=self.top_k,
+ add_profile_id=True,
+ add_history_id=True,
+ )
+ profile_nodes.extend(nodes)
+
+ seen_ids = {node.memory_id: node for node in self.retrieved_nodes if node.memory_id}
+ new_nodes = []
+ for node in profile_nodes:
+ if node.memory_id not in seen_ids:
+ seen_ids[node.memory_id] = node
+ new_nodes.append(node)
+ self.retrieved_nodes.extend(new_nodes)
+
+ if not new_nodes:
+ output = "No new profiles found."
+ else:
+ output = "\n".join(
+ [ProfileHandler.format_node(node, add_profile_id=True, add_history_id=True) for node in new_nodes],
+ )
+
+ logger.info(f"Retrieved {len(profile_nodes)} profiles, {len(new_nodes)} new after deduplication")
+ return output
diff --git a/reme/memory/vector_tools/profiles/vector_profile_backend.py b/reme/memory/vector_tools/profiles/vector_profile_backend.py
new file mode 100644
index 00000000..3147cc92
--- /dev/null
+++ b/reme/memory/vector_tools/profiles/vector_profile_backend.py
@@ -0,0 +1,55 @@
+"""Vector-backed profile storage."""
+
+from .profile_backend import BaseProfileBackend
+from .profile_vector_handler import ProfileVectorHandler
+from ....core import ServiceContext
+from ....core.schema import MemoryNode
+
+
+class VectorProfileBackend(BaseProfileBackend):
+ """Persist user profiles in a dedicated vector store."""
+
+ def __init__(
+ self,
+ memory_target: str,
+ service_context: ServiceContext,
+ vector_store_name: str = "profile",
+ max_capacity: int = 50,
+ ):
+ super().__init__(memory_target=memory_target, max_capacity=max_capacity)
+ self.handler = ProfileVectorHandler(
+ memory_target=memory_target,
+ service_context=service_context,
+ vector_store_name=vector_store_name,
+ max_capacity=max_capacity,
+ )
+
+ async def get_all(self) -> list[MemoryNode]:
+ return await self.handler.get_all()
+
+ async def get_by(self, *, profile_id: str | None = None, profile_key: str | None = None) -> MemoryNode | None:
+ return await self.handler.get_by(profile_id=profile_id, profile_key=profile_key)
+
+ async def delete(self, profile_id: str | list[str]) -> bool | int:
+ return await self.handler.delete(profile_id)
+
+ async def delete_all(self) -> int:
+ return await self.handler.delete_all()
+
+ async def add(self, message_time: str, profile_key: str, profile_value: str, ref_memory_id: str = "") -> MemoryNode:
+ return await self.handler.add(message_time, profile_key, profile_value, ref_memory_id)
+
+ async def add_batch(self, profiles: list[dict], ref_memory_id: str = "") -> list[MemoryNode]:
+ return await self.handler.add_batch(profiles, ref_memory_id)
+
+ async def update(
+ self,
+ profile_id: str,
+ message_time: str,
+ profile_key: str,
+ profile_value: str,
+ ) -> MemoryNode | None:
+ return await self.handler.update(profile_id, message_time, profile_key, profile_value)
+
+ async def search(self, query: str | list[str], limit: int = 5) -> list[MemoryNode]:
+ return await self.handler.search(query, limit)
From f42cf60706611afdfa57dd3c09bca106c93c368f Mon Sep 17 00:00:00 2001
From: lichen2015
Date: Fri, 8 May 2026 17:12:11 +0800
Subject: [PATCH 05/16] add zvec vector/file store (#218)
---
README.md | 2 +-
README_ZH.md | 2 +-
docs/vector_store_api_guide.md | 33 +-
reme/core/file_store/__init__.py | 3 +
reme/core/file_store/zvec_file_store.py | 573 +++++++++++++
reme/core/vector_store/__init__.py | 3 +
reme/core/vector_store/zvec_vector_store.py | 809 +++++++++++++++++
tests/test_file_store.py | 45 +-
tests/test_vector_store.py | 39 +-
tests/test_zvec_vector_store.py | 906 ++++++++++++++++++++
tests/vector/test_reme_vector.py | 2 +-
11 files changed, 2409 insertions(+), 8 deletions(-)
create mode 100644 reme/core/file_store/zvec_file_store.py
create mode 100644 reme/core/vector_store/zvec_vector_store.py
create mode 100644 tests/test_zvec_vector_store.py
diff --git a/README.md b/README.md
index f2f413c0..b4bb48c4 100644
--- a/README.md
+++ b/README.md
@@ -506,7 +506,7 @@ async def main():
"dimensions": 1024,
},
default_vector_store_config={
- "backend": "local", # Supports local/chroma/qdrant/elasticsearch/obvec
+ "backend": "local", # Supports local/chroma/qdrant/elasticsearch/obvec/zvec
},
)
await reme.start()
diff --git a/README_ZH.md b/README_ZH.md
index 7e6ccd58..210a11ce 100644
--- a/README_ZH.md
+++ b/README_ZH.md
@@ -486,7 +486,7 @@ async def main():
"dimensions": 1024,
},
default_vector_store_config={
- "backend": "local", # 支持 local/chroma/qdrant/elasticsearch/obvec
+ "backend": "local", # 支持 local/chroma/qdrant/elasticsearch/obvec/zvec
},
)
await reme.start()
diff --git a/docs/vector_store_api_guide.md b/docs/vector_store_api_guide.md
index b0a06eab..89881ef2 100644
--- a/docs/vector_store_api_guide.md
+++ b/docs/vector_store_api_guide.md
@@ -34,6 +34,7 @@ FlowLLM provides multiple Vector Store implementations tailored to different use
- **ChromaVectorStore** ([source code](https://github.com/flowllm-ai/flowllm/blob/main/flowllm/core/vector_store/chroma_vector_store.py)): Based on ChromaDB, providing persistent storage and metadata filtering capabilities.
- **EsVectorStore** ([source code](https://github.com/flowllm-ai/flowllm/blob/main/flowllm/core/vector_store/es_vector_store.py)): Built on Elasticsearch, enabling powerful combined full-text and vector search functionalities.
- **ObVecVectorStore** ([source code](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/obvec_vector_store.py)): Uses [pyobvector](https://pypi.org/project/pyobvector/) against **OceanBase** or **seekdb** (MySQL-compatible wire protocol). Suitable when you already run OceanBase/seekdb or need a SQL-native vector table with HNSW-style ANN search and JSON metadata filters.
+- **ZvecVectorStore** ([source code](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/zvec_vector_store.py)): Built on zvec, a high-performance local vector database with strong-schema support and HNSW indexing. Suitable for single-machine deployments requiring fast vector search.
All Vector Store implementations inherit from **BaseVectorStore** ([source code](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/base_vector_store.py)) in ReMe, ensuring a consistent interface specification.
@@ -130,6 +131,11 @@ docker run -d --name reme_seekdb -p 2881:2881 -e ROOT_PASSWORD= python tests/test_vector_store.py --obvec
```
+### ZvecVectorStore Configuration
+
+- **db_path**: Local storage path for persistent mode (required).
+- **dimension**: Dimensionality of the embedding vectors (default: `1024`).
+- **distance**: Distance metric — supports `cosine`, `l2`, `ip` (default: `cosine`).
## Configuration File Examples
@@ -151,7 +157,7 @@ vector_store.default.params.=
### Configuration Field Descriptions
-- **`backend`** (required): Vector store backend type. Options: `local`, `memory`, `chroma`, `qdrant`, `elasticsearch`, `obvec`.
+- **`backend`** (required): Vector store backend type. Options: `local`, `memory`, `chroma`, `qdrant`, `elasticsearch`, `obvec`, `zvec`.
- **`embedding_model`** (required): Name of the embedding model configuration, referencing the `embedding_model` section.
- **`params`** (optional): Dictionary of backend-specific parameters passed to the vector store constructor.
@@ -347,6 +353,30 @@ vector_stores.default.password=your-root-password
ReMe service YAML uses the key `vector_stores` (plural); CLI overrides use the same nested paths.
+#### 7. ZvecVectorStore Configuration
+
+Persistent local storage based on zvec with HNSW indexing and strong-schema support.
+
+**Implementation**: [`reme/core/vector_store/zvec_vector_store.py`](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/zvec_vector_store.py)
+
+```yaml
+vector_store:
+ default:
+ backend: zvec
+ embedding_model: default
+ params:
+ db_path: "./zvec_vector_store" # Local storage path (required)
+ dimension: 1024 # Vector dimension (optional; default: 1024)
+ distance: "cosine" # Distance metric (optional; default: cosine; options: cosine, l2, ip)
+```
+
+```shell
+vector_store.default.backend=zvec
+vector_store.default.params.db_path=./zvec_vector_store
+vector_store.default.params.dimension=1024
+vector_store.default.params.distance=cosine
+```
+
### Complete Configuration Example
Below is a complete `default.yaml` example including both embedding model and vector store configurations:
@@ -405,6 +435,7 @@ Two types of metadata filtering are supported:
- **Development & Testing**: Use MemoryVectorStore or LocalVectorStore—no additional services required.
- **Small-Scale Applications**: Use LocalVectorStore or ChromaVectorStore for simplicity and ease of use.
- **Production Environments**: Use QdrantVectorStore, EsVectorStore, or ObVecVectorStore (OceanBase/seekdb) for high performance and scalability, depending on your existing infrastructure.
+- **High-Performance Local Search**: Use ZvecVectorStore for single-machine deployments requiring fast HNSW-based vector search with local persistence.
- **Hybrid Search**: Use EsVectorStore to combine vector search with full-text search capabilities.
- **OceanBase / seekdb**: Use ObVecVectorStore when you standardize on pyobvector and SQL-accessible vector tables.
diff --git a/reme/core/file_store/__init__.py b/reme/core/file_store/__init__.py
index 1358df52..a8457406 100644
--- a/reme/core/file_store/__init__.py
+++ b/reme/core/file_store/__init__.py
@@ -9,6 +9,7 @@ from .base_file_store import BaseFileStore
from .chroma_file_store import ChromaFileStore
from .local_file_store import LocalFileStore
from .sqlite_file_store import SqliteFileStore
+from .zvec_file_store import ZvecFileStore
from ..registry_factory import R
__all__ = [
@@ -16,8 +17,10 @@ __all__ = [
"ChromaFileStore",
"LocalFileStore",
"SqliteFileStore",
+ "ZvecFileStore",
]
R.file_stores.register("sqlite")(SqliteFileStore)
R.file_stores.register("chroma")(ChromaFileStore)
R.file_stores.register("local")(LocalFileStore)
+R.file_stores.register("zvec")(ZvecFileStore)
diff --git a/reme/core/file_store/zvec_file_store.py b/reme/core/file_store/zvec_file_store.py
new file mode 100644
index 00000000..3d162819
--- /dev/null
+++ b/reme/core/file_store/zvec_file_store.py
@@ -0,0 +1,573 @@
+"""Zvec storage backend for file store."""
+
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+from typing import Any
+
+from .base_file_store import BaseFileStore
+from ..enumeration import MemorySource
+from ..schema import FileMetadata, MemoryChunk, MemorySearchResult
+from ..utils import get_logger
+
+logger = get_logger()
+
+_ZVEC_IMPORT_ERROR: Exception | None = None
+
+try:
+ import zvec # type: ignore[import-untyped]
+ from zvec import (
+ CollectionOption,
+ CollectionSchema,
+ DataType,
+ Doc,
+ FieldSchema,
+ HnswIndexParam,
+ InvertIndexParam,
+ VectorQuery,
+ VectorSchema,
+ )
+ from zvec.typing import MetricType
+except Exception as e:
+ _ZVEC_IMPORT_ERROR = e
+ zvec = None # type: ignore[assignment]
+
+
+# zvec max topk (will be lifted to 100,000 in zvec v0.3.2+)
+_ZVEC_MAX_TOPK = 1024
+
+# Default vector field name
+_DEFAULT_VECTOR_FIELD = "embedding"
+
+
+def _escape(value: str) -> str:
+ """Escape a string value for zvec filter expressions."""
+ return value.replace("'", "\\'")
+
+
+def _build_file_store_schema(name: str, dimension: int) -> CollectionSchema:
+ """Build a zvec CollectionSchema for file store chunks."""
+ return CollectionSchema(
+ name=name,
+ fields=[
+ FieldSchema("content", DataType.STRING, nullable=True, index_param=InvertIndexParam()),
+ FieldSchema("path", DataType.STRING, nullable=True, index_param=InvertIndexParam()),
+ FieldSchema("source", DataType.STRING, nullable=True, index_param=InvertIndexParam()),
+ FieldSchema("start_line", DataType.INT64, nullable=True),
+ FieldSchema("end_line", DataType.INT64, nullable=True),
+ FieldSchema("hash", DataType.STRING, nullable=True),
+ FieldSchema("updated_at", DataType.INT64, nullable=True),
+ FieldSchema("file_metadata", DataType.STRING, nullable=True),
+ ],
+ vectors=[
+ VectorSchema(
+ name=_DEFAULT_VECTOR_FIELD,
+ data_type=DataType.VECTOR_FP32,
+ dimension=dimension,
+ index_param=HnswIndexParam(metric_type=MetricType.COSINE),
+ ),
+ ],
+ )
+
+
+def _chunk_to_doc(chunk: MemoryChunk, file_meta_json: str = "{}") -> Doc:
+ """Convert a MemoryChunk to a zvec Doc."""
+ fields: dict[str, Any] = {
+ "content": chunk.text,
+ "path": chunk.path,
+ "source": chunk.source.value if chunk.source else "",
+ "start_line": chunk.start_line,
+ "end_line": chunk.end_line,
+ "hash": chunk.hash,
+ "updated_at": int(time.time() * 1000),
+ "file_metadata": file_meta_json,
+ }
+ vectors: dict[str, Any] = {}
+ if chunk.embedding is not None:
+ vectors[_DEFAULT_VECTOR_FIELD] = chunk.embedding
+ return Doc(id=chunk.id, fields=fields, vectors=vectors)
+
+
+def _doc_to_chunk(doc: Doc) -> MemoryChunk:
+ """Convert a zvec Doc to a MemoryChunk."""
+ raw_vector = doc.vector(_DEFAULT_VECTOR_FIELD)
+ vector = raw_vector if isinstance(raw_vector, list) and len(raw_vector) > 0 else None
+ return MemoryChunk(
+ id=str(doc.id),
+ path=str(doc.field("path") or ""),
+ source=MemorySource(str(doc.field("source") or "")),
+ start_line=int(doc.field("start_line") or 0),
+ end_line=int(doc.field("end_line") or 0),
+ text=str(doc.field("content") or ""),
+ hash=str(doc.field("hash") or ""),
+ embedding=vector,
+ )
+
+
+def _build_source_filter(sources: list[MemorySource] | None) -> str | None:
+ """Build a zvec filter expression for source filtering."""
+ if not sources:
+ return None
+ if len(sources) == 1:
+ return f"source='{_escape(sources[0].value)}'"
+ vals = ", ".join(f"'{_escape(s.value)}'" for s in sources)
+ return f"source IN ({vals})"
+
+
+class ZvecFileStore(BaseFileStore):
+ """Zvec file storage with vector and keyword search.
+
+ Provides zvec-backed persistent storage with:
+ - Vector similarity search (native zvec HNSW)
+ - Keyword search (Python substring matching on fetched results)
+ - Hybrid search (weighted fusion of vector and keyword results)
+
+ Note:
+ Keyword search operates on chunks fetched from zvec, which is subject
+ to the topk limit (1024 in zvec < v0.3.2, 100,000 in v0.3.2+).
+ For collections with more chunks than the topk limit, keyword search
+ may not scan all documents.
+ """
+
+ def __init__(
+ self,
+ store_name: str,
+ db_path: str | Path,
+ embedding_model: Any | None = None,
+ vector_enabled: bool = False,
+ fts_enabled: bool = True,
+ dimension: int = 1024,
+ **kwargs: Any,
+ ):
+ if _ZVEC_IMPORT_ERROR is not None:
+ raise ImportError(
+ "Zvec requires extra dependencies. Install with `pip install zvec`",
+ ) from _ZVEC_IMPORT_ERROR
+
+ super().__init__(
+ store_name=store_name,
+ db_path=db_path,
+ embedding_model=embedding_model,
+ vector_enabled=vector_enabled,
+ fts_enabled=fts_enabled,
+ **kwargs,
+ )
+
+ self.dimension = dimension
+ self._collection = None
+ self._initialized = False
+ self._metadata_file: Path = self.db_path / f"{store_name}_file_metadata.json"
+ self._metadata_cache: dict[str, dict[str, FileMetadata]] = {}
+
+ @property
+ def collection_name(self) -> str:
+ """Get the name of the zvec collection for this store."""
+ return f"chunks_{self.store_name}"
+
+ # ------------------------------------------------------------------
+ # Lifecycle
+ # ------------------------------------------------------------------
+
+ async def start(self) -> None:
+ """Initialize zvec engine and open the collection."""
+ if not self._initialized:
+ try:
+ zvec.init()
+ except RuntimeError:
+ pass
+ self._initialized = True
+
+ self.db_path.mkdir(parents=True, exist_ok=True)
+ collection_path = str(self.db_path / self.collection_name)
+ option = CollectionOption(read_only=False, enable_mmap=True)
+
+ try:
+ self._collection = zvec.open(collection_path, option)
+ logger.info(f"Opened existing zvec file store collection: {collection_path}")
+ except Exception:
+ schema = _build_file_store_schema(self.collection_name, self.dimension)
+ self._collection = zvec.create_and_open(
+ path=collection_path,
+ schema=schema,
+ option=option,
+ )
+ logger.info(f"Created new zvec file store collection: {collection_path}")
+
+ self._metadata_cache = await self._load_metadata()
+
+ async def close(self) -> None:
+ """Close zvec collection and persist metadata."""
+ if self._metadata_cache:
+ await self._save_metadata(self._metadata_cache)
+
+ if self._collection is not None:
+ try:
+ self._collection.flush()
+ except Exception as e:
+ logger.warning(f"Failed to flush collection on close: {e}")
+ self._collection = None
+
+ # ------------------------------------------------------------------
+ # Metadata management
+ # ------------------------------------------------------------------
+
+ async def _load_metadata(self) -> dict[str, dict[str, FileMetadata]]:
+ """Load file metadata from JSON file."""
+ if not self._metadata_file.exists():
+ return {}
+ try:
+ data = json.loads(self._metadata_file.read_text(encoding="utf-8"))
+ result: dict[str, dict[str, FileMetadata]] = {}
+ for source, files in data.items():
+ result[source] = {}
+ for path, meta in files.items():
+ result[source][path] = FileMetadata(**meta)
+ return result
+ except Exception as e:
+ logger.warning(f"Failed to load metadata from {self._metadata_file}: {e}")
+ return {}
+
+ async def _save_metadata(self, metadata: dict[str, dict[str, FileMetadata]]) -> None:
+ """Save file metadata to JSON file."""
+ try:
+ out: dict[str, dict[str, dict]] = {}
+ for source, files in metadata.items():
+ out[source] = {}
+ for path, meta in files.items():
+ out[source][path] = {
+ "path": meta.path,
+ "hash": meta.hash,
+ "mtime_ms": meta.mtime_ms,
+ "size": meta.size,
+ "chunk_count": meta.chunk_count,
+ }
+ self._metadata_file.write_text(
+ json.dumps(out, indent=2, ensure_ascii=False),
+ encoding="utf-8",
+ )
+ except Exception as e:
+ logger.error(f"Failed to save metadata to {self._metadata_file}: {e}")
+
+ # ------------------------------------------------------------------
+ # CRUD operations
+ # ------------------------------------------------------------------
+
+ async def upsert_file(
+ self,
+ file_meta: FileMetadata,
+ source: MemorySource,
+ chunks: list[MemoryChunk],
+ ) -> None:
+ """Insert or update a file and its chunks."""
+ if not chunks:
+ return
+
+ # Delete existing chunks for this file first
+ await self.delete_file(file_meta.path, source)
+
+ # Generate embeddings
+ chunks = await self.get_chunk_embeddings(chunks)
+
+ file_meta_json = json.dumps(
+ {
+ "path": file_meta.path,
+ "hash": file_meta.hash,
+ "mtime_ms": file_meta.mtime_ms,
+ "size": file_meta.size,
+ "chunk_count": len(chunks),
+ },
+ ensure_ascii=False,
+ )
+
+ docs = [_chunk_to_doc(c, file_meta_json) for c in chunks]
+ self._collection.insert(docs)
+
+ # Update metadata cache
+ if source.value not in self._metadata_cache:
+ self._metadata_cache[source.value] = {}
+ self._metadata_cache[source.value][file_meta.path] = FileMetadata(
+ hash=file_meta.hash,
+ mtime_ms=file_meta.mtime_ms,
+ size=file_meta.size,
+ path=file_meta.path,
+ chunk_count=len(chunks),
+ )
+
+ async def delete_file(self, path: str, source: MemorySource) -> None:
+ """Delete a file and all its chunks."""
+ filter_expr = f"path='{_escape(path)}' AND source='{_escape(source.value)}'"
+ results = self._collection.query(topk=_ZVEC_MAX_TOPK, filter=filter_expr, include_vector=False)
+
+ ids_to_delete = [doc.id for doc in results]
+ if ids_to_delete:
+ self._collection.delete(ids_to_delete)
+
+ if source.value in self._metadata_cache:
+ self._metadata_cache[source.value].pop(path, None)
+
+ async def delete_file_chunks(self, path: str, chunk_ids: list[str]) -> None:
+ """Delete specific chunks for a file."""
+ if not chunk_ids:
+ return
+ self._collection.delete(chunk_ids)
+
+ async def upsert_chunks(
+ self,
+ chunks: list[MemoryChunk],
+ source: MemorySource,
+ ) -> None:
+ """Insert or update specific chunks."""
+ if not chunks:
+ return
+
+ chunks = await self.get_chunk_embeddings(chunks)
+ docs = [_chunk_to_doc(c) for c in chunks]
+ self._collection.upsert(docs)
+
+ # ------------------------------------------------------------------
+ # Listing and metadata
+ # ------------------------------------------------------------------
+
+ async def list_files(self, source: MemorySource) -> list[str]:
+ """List all indexed files for a source."""
+ if source.value not in self._metadata_cache:
+ return []
+ return list(self._metadata_cache[source.value].keys())
+
+ async def get_file_metadata(
+ self,
+ path: str,
+ source: MemorySource,
+ ) -> FileMetadata | None:
+ """Get file metadata."""
+ if source.value not in self._metadata_cache:
+ return None
+ return self._metadata_cache[source.value].get(path)
+
+ async def update_file_metadata(self, file_meta: FileMetadata, source: MemorySource) -> None:
+ """Update file metadata without affecting chunks."""
+ if source.value not in self._metadata_cache:
+ self._metadata_cache[source.value] = {}
+ self._metadata_cache[source.value][file_meta.path] = FileMetadata(
+ hash=file_meta.hash,
+ mtime_ms=file_meta.mtime_ms,
+ size=file_meta.size,
+ path=file_meta.path,
+ chunk_count=file_meta.chunk_count,
+ )
+
+ async def get_file_chunks(
+ self,
+ path: str,
+ source: MemorySource,
+ ) -> list[MemoryChunk]:
+ """Get all chunks for a file."""
+ filter_expr = f"path='{_escape(path)}' AND source='{_escape(source.value)}'"
+ results = self._collection.query(
+ topk=_ZVEC_MAX_TOPK,
+ filter=filter_expr,
+ include_vector=True,
+ )
+ chunks = [_doc_to_chunk(doc) for doc in results]
+ chunks.sort(key=lambda c: c.start_line)
+ return chunks
+
+ # ------------------------------------------------------------------
+ # Search
+ # ------------------------------------------------------------------
+
+ async def vector_search(
+ self,
+ query: str,
+ limit: int,
+ sources: list[MemorySource] | None = None,
+ ) -> list[MemorySearchResult]:
+ """Perform vector similarity search."""
+ if not self.vector_enabled or not query:
+ return []
+
+ query_embedding = await self.get_embedding(query)
+ if not query_embedding:
+ return []
+
+ filter_expr = _build_source_filter(sources)
+ vq = VectorQuery(field_name=_DEFAULT_VECTOR_FIELD, vector=query_embedding)
+
+ try:
+ results = self._collection.query(
+ vectors=vq,
+ topk=min(limit, _ZVEC_MAX_TOPK),
+ filter=filter_expr,
+ include_vector=False,
+ )
+ except Exception as e:
+ logger.error(f"Vector search failed: {e}")
+ return []
+
+ search_results = []
+ for doc in results:
+ score = doc.score if doc.score is not None else 0.0
+ # zvec cosine score might need normalization depending on version
+ search_results.append(
+ MemorySearchResult(
+ path=str(doc.field("path") or ""),
+ start_line=int(doc.field("start_line") or 0),
+ end_line=int(doc.field("end_line") or 0),
+ score=score,
+ snippet=str(doc.field("content") or ""),
+ source=MemorySource(str(doc.field("source") or "")),
+ raw_metric=score,
+ ),
+ )
+
+ search_results.sort(key=lambda r: r.score, reverse=True)
+ return search_results[:limit]
+
+ async def keyword_search(
+ self,
+ query: str,
+ limit: int,
+ sources: list[MemorySource] | None = None,
+ ) -> list[MemorySearchResult]:
+ """Perform keyword search via Python substring matching.
+
+ Fetches chunks from zvec (subject to topk limit) then matches
+ keywords in Python. For collections larger than the topk limit,
+ not all documents are scanned.
+ """
+ if not self.fts_enabled or not query:
+ return []
+
+ words = query.split()
+ if not words:
+ return []
+
+ # Fetch candidate chunks from zvec
+ filter_expr = _build_source_filter(sources)
+ results = self._collection.query(
+ topk=_ZVEC_MAX_TOPK,
+ filter=filter_expr,
+ include_vector=False,
+ )
+
+ query_lower = query.lower()
+ words_lower = [w.lower() for w in words]
+ n_words = len(words)
+
+ search_results = []
+ for doc in results:
+ text = str(doc.field("content") or "")
+ text_lower = text.lower()
+ match_count = sum(1 for w in words_lower if w in text_lower)
+ if match_count == 0:
+ continue
+
+ base_score = match_count / n_words
+ phrase_bonus = 0.2 if n_words > 1 and query_lower in text_lower else 0.0
+ score = min(1.0, base_score + phrase_bonus)
+
+ search_results.append(
+ MemorySearchResult(
+ path=str(doc.field("path") or ""),
+ start_line=int(doc.field("start_line") or 0),
+ end_line=int(doc.field("end_line") or 0),
+ score=score,
+ snippet=text,
+ source=MemorySource(str(doc.field("source") or "")),
+ ),
+ )
+
+ search_results.sort(key=lambda r: r.score, reverse=True)
+ return search_results[:limit]
+
+ async def hybrid_search(
+ self,
+ query: str,
+ limit: int,
+ sources: list[MemorySource] | None = None,
+ vector_weight: float = 0.7,
+ candidate_multiplier: float = 3.0,
+ ) -> list[MemorySearchResult]:
+ """Perform hybrid search combining vector and keyword search."""
+ assert 0.0 <= vector_weight <= 1.0, f"vector_weight must be between 0 and 1, got {vector_weight}"
+
+ candidates = min(200, max(1, int(limit * candidate_multiplier)))
+ text_weight = 1.0 - vector_weight
+
+ if self.vector_enabled and self.fts_enabled:
+ keyword_results = await self.keyword_search(query, candidates, sources)
+ vector_results = await self.vector_search(query, candidates, sources)
+
+ if not keyword_results:
+ return vector_results[:limit]
+ elif not vector_results:
+ return keyword_results[:limit]
+ else:
+ return self._merge_hybrid_results(
+ vector=vector_results,
+ keyword=keyword_results,
+ vector_weight=vector_weight,
+ text_weight=text_weight,
+ )[:limit]
+ elif self.vector_enabled:
+ return await self.vector_search(query, limit, sources)
+ elif self.fts_enabled:
+ return await self.keyword_search(query, limit, sources)
+ else:
+ return []
+
+ @staticmethod
+ def _merge_hybrid_results(
+ vector: list[MemorySearchResult],
+ keyword: list[MemorySearchResult],
+ vector_weight: float,
+ text_weight: float,
+ ) -> list[MemorySearchResult]:
+ """Merge vector and keyword search results with weighted scoring."""
+ merged: dict[str, MemorySearchResult] = {}
+
+ for result in vector:
+ result.score = result.score * vector_weight
+ merged[result.merge_key] = result
+
+ for result in keyword:
+ key = result.merge_key
+ if key in merged:
+ merged[key].score += result.score * text_weight
+ else:
+ result.score = result.score * text_weight
+ merged[key] = result
+
+ results = list(merged.values())
+ results.sort(key=lambda r: r.score, reverse=True)
+ return results
+
+ # ------------------------------------------------------------------
+ # Maintenance
+ # ------------------------------------------------------------------
+
+ async def clear_all(self) -> None:
+ """Clear all indexed data."""
+ # Delete all documents
+ stats = self._collection.stats
+ count = stats.doc_count if stats else 0
+ if count > 0:
+ try:
+ self._collection.delete_by_filter("content!=''")
+ except Exception:
+ remaining = count
+ while remaining > 0:
+ batch = self._collection.query(
+ topk=min(remaining, _ZVEC_MAX_TOPK),
+ include_vector=False,
+ )
+ if not batch:
+ break
+ self._collection.delete([doc.id for doc in batch])
+ remaining -= len(batch)
+
+ self._metadata_cache = {}
+ await self._save_metadata({})
+ logger.info(f"Cleared all data from zvec file store: {self.collection_name}")
diff --git a/reme/core/vector_store/__init__.py b/reme/core/vector_store/__init__.py
index 8429b911..0426fd64 100644
--- a/reme/core/vector_store/__init__.py
+++ b/reme/core/vector_store/__init__.py
@@ -7,6 +7,7 @@ from .local_vector_store import LocalVectorStore
from .obvec_vector_store import ObVecVectorStore
from .pgvector_store import PGVectorStore
from .qdrant_vector_store import QdrantVectorStore
+from .zvec_vector_store import ZvecVectorStore
from ..registry_factory import R
__all__ = [
@@ -17,6 +18,7 @@ __all__ = [
"ObVecVectorStore",
"PGVectorStore",
"QdrantVectorStore",
+ "ZvecVectorStore",
]
R.vector_stores.register("chroma")(ChromaVectorStore)
@@ -25,3 +27,4 @@ R.vector_stores.register("local")(LocalVectorStore)
R.vector_stores.register("obvec")(ObVecVectorStore)
R.vector_stores.register("pgvector")(PGVectorStore)
R.vector_stores.register("qdrant")(QdrantVectorStore)
+R.vector_stores.register("zvec")(ZvecVectorStore)
diff --git a/reme/core/vector_store/zvec_vector_store.py b/reme/core/vector_store/zvec_vector_store.py
new file mode 100644
index 00000000..dde12a0b
--- /dev/null
+++ b/reme/core/vector_store/zvec_vector_store.py
@@ -0,0 +1,809 @@
+"""Zvec vector store implementation for the ReMe framework."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+
+from .base_vector_store import BaseVectorStore
+from ..embedding import BaseEmbeddingModel
+from ..schema import VectorNode
+
+_ZVEC_IMPORT_ERROR: Exception | None = None
+
+try:
+ import zvec # type: ignore[import-untyped]
+ from zvec import (
+ CollectionOption,
+ CollectionSchema,
+ DataType,
+ Doc,
+ FieldSchema,
+ HnswIndexParam,
+ InvertIndexParam,
+ VectorQuery,
+ VectorSchema,
+ )
+ from zvec.typing import MetricType
+except Exception as e:
+ _ZVEC_IMPORT_ERROR = e
+ zvec = None # type: ignore[assignment]
+
+
+# Default vector field name used inside zvec collections
+_DEFAULT_VECTOR_FIELD = "embedding"
+
+# Default scalar content field name for storing text
+_CONTENT_FIELD = "content"
+
+# Field name for JSON-serialized metadata
+_METADATA_FIELD = "metadata"
+
+# Metadata fields promoted to top-level zvec schema columns for native filtering.
+# These are the most commonly filtered keys in ReMe's memory system.
+# Defining them as independent schema columns allows zvec to perform
+# filtering at the database level instead of Python post-filtering.
+# Format: {metadata_key: (zvec_data_type_str, has_inverted_index)}
+_PROMOTED_FIELD_SPECS: dict[str, tuple[str, bool]] = {
+ "memory_type": ("STRING", True), # Inverted index for exact match filtering
+ "memory_target": ("STRING", True), # Inverted index for exact match filtering
+ "author": ("STRING", False),
+ "time_int": ("INT64", False), # Numeric for range queries
+}
+
+# zvec data-type string → DataType enum mapping (populated after import)
+_DATATYPE_MAP: dict[str, Any] = {} # filled in _build_collection_schema
+
+
+def _escape_zvec_string(value: str) -> str:
+ """Escape a string value for use in zvec filter expressions."""
+ return value.replace("'", "\\'")
+
+
+def _build_zvec_filter(
+ filters: dict | None,
+ promoted_fields: set[str],
+) -> tuple[str | None, dict | None]:
+ """Split ReMe filter dict into a zvec native filter expression and remaining post-filters.
+
+ For filter keys that correspond to promoted schema fields, native
+ zvec filter expressions are generated. Non-promoted keys are
+ kept for Python post-filtering.
+
+ Args:
+ filters: ReMe-style filter dictionary.
+ promoted_fields: Set of metadata keys that exist as top-level schema columns.
+
+ Returns:
+ (native_filter_expr, post_filter_dict) — either may be None.
+ """
+ if not filters:
+ return None, None
+
+ native_conditions: list[str] = []
+ post_filters: dict = {}
+
+ for key, value in filters.items():
+ if key.startswith("$"):
+ # Compound operators ($or, $and, $not) — keep for post-filtering
+ post_filters[key] = value
+ continue
+
+ if key not in promoted_fields:
+ # Not a promoted field — use post-filtering
+ post_filters[key] = value
+ continue
+
+ # Build native filter condition for promoted fields
+ field_type = _PROMOTED_FIELD_SPECS.get(key, ("STRING", False))[0]
+
+ if isinstance(value, list) and len(value) == 2:
+ # Range query: [start, end]
+ if field_type == "INT64":
+ native_conditions.append(f"{key} >= {value[0]} AND {key} <= {value[1]}")
+ else:
+ # STRING range — use >= and <= with string escaping
+ native_conditions.append(
+ f"{key} >= '{_escape_zvec_string(str(value[0]))}' "
+ f"AND {key} <= '{_escape_zvec_string(str(value[1]))}'",
+ )
+ elif isinstance(value, bool):
+ native_conditions.append(f"{key} = {str(value).upper()}")
+ elif isinstance(value, (int, float)):
+ native_conditions.append(f"{key} = {value}")
+ elif isinstance(value, str):
+ native_conditions.append(f"{key} = '{_escape_zvec_string(value)}'")
+ else:
+ # Unsupported type — fall back to post-filtering
+ post_filters[key] = value
+
+ native_filter = " AND ".join(native_conditions) if native_conditions else None
+ return native_filter, post_filters if post_filters else None
+
+
+def _metric_type_from_str(metric: str) -> Any:
+ """Convert a string metric name to zvec MetricType enum value."""
+ if zvec is None:
+ return None
+ mapping = {
+ "cosine": MetricType.COSINE,
+ "l2": MetricType.L2,
+ "ip": MetricType.IP,
+ }
+ return mapping.get(metric.lower(), MetricType.COSINE)
+
+
+def _build_collection_schema(
+ name: str,
+ dimension: int,
+ metric: str = "cosine",
+) -> CollectionSchema:
+ """Build a zvec CollectionSchema for ReMe usage.
+
+ The schema contains:
+ - "content" (STRING, inverted index) — text content
+ - "metadata" (STRING) — JSON-serialized metadata dictionary
+ - Promoted metadata fields (STRING / INT64) — for native zvec filtering
+ - "embedding" (VECTOR_FP32, dimension, HNSW index) — the vector field
+
+ Promoted fields are commonly filtered metadata keys defined as top-level
+ schema columns so that zvec can perform filtering natively instead of
+ Python post-filtering. The full metadata is still stored as JSON in the
+ "metadata" field for complete round-trip serialization.
+
+ zvec automatically manages the document ID (string type); we do NOT
+ define an "id" field in the schema.
+ """
+ # Populate the DataType map on first call
+ if not _DATATYPE_MAP:
+ _DATATYPE_MAP.update(
+ {
+ "STRING": DataType.STRING,
+ "INT64": DataType.INT64,
+ },
+ )
+
+ distance = _metric_type_from_str(metric)
+
+ # Base fields
+ fields = [
+ FieldSchema("content", DataType.STRING, nullable=True, index_param=InvertIndexParam()),
+ FieldSchema("metadata", DataType.STRING, nullable=True),
+ ]
+
+ # Add promoted metadata fields as top-level schema columns
+ for field_name, (type_str, has_inv_index) in _PROMOTED_FIELD_SPECS.items():
+ dt = _DATATYPE_MAP[type_str]
+ idx_param = InvertIndexParam() if has_inv_index else None
+ fields.append(FieldSchema(field_name, dt, nullable=True, index_param=idx_param))
+
+ return CollectionSchema(
+ name=name,
+ fields=fields,
+ vectors=[
+ VectorSchema(
+ name=_DEFAULT_VECTOR_FIELD,
+ data_type=DataType.VECTOR_FP32,
+ dimension=dimension,
+ index_param=HnswIndexParam(metric_type=distance),
+ ),
+ ],
+ )
+
+
+def _vector_node_to_doc(node: VectorNode) -> Doc:
+ """Convert a ReMe VectorNode to a zvec Doc.
+
+ Metadata is serialized as a JSON string into the "metadata" field.
+ The "score" key is excluded since it is a computed value, not stored data.
+ Promoted metadata fields are also extracted as top-level Doc fields
+ for native zvec filtering.
+ The vector is placed under the default vector field name.
+ The zvec Doc id must be a string.
+ """
+ # Filter out computed score before serialization
+ meta_to_store = {k: v for k, v in node.metadata.items() if k != "score"}
+
+ fields: dict[str, Any] = {
+ "content": node.content,
+ "metadata": json.dumps(meta_to_store) if meta_to_store else "{}",
+ }
+
+ # Extract promoted metadata fields as top-level schema columns
+ for field_name, (type_str, _) in _PROMOTED_FIELD_SPECS.items():
+ value = meta_to_store.get(field_name)
+ if value is not None:
+ # Ensure correct type: INT64 fields must be int
+ if type_str == "INT64" and not isinstance(value, int):
+ try:
+ value = int(value)
+ except (ValueError, TypeError):
+ continue
+ fields[field_name] = value
+
+ vectors: dict[str, Any] = {}
+ if node.vector is not None:
+ vectors[_DEFAULT_VECTOR_FIELD] = node.vector
+
+ return Doc(id=str(node.vector_id), fields=fields, vectors=vectors)
+
+
+def _doc_to_vector_node(doc: Doc, include_score: bool = False) -> VectorNode:
+ """Convert a zvec Doc back to a ReMe VectorNode.
+
+ The "metadata" field is parsed from JSON. The "content" field becomes
+ the node content. If ``include_score`` is True, the search score is
+ added to the metadata dictionary.
+ """
+ metadata: dict[str, str | bool | int | float] = {}
+
+ # Parse JSON metadata
+ raw_metadata = doc.field("metadata")
+ if raw_metadata:
+ try:
+ parsed = json.loads(raw_metadata)
+ if isinstance(parsed, dict):
+ metadata.update(parsed)
+ except (json.JSONDecodeError, TypeError):
+ logger.warning(f"Failed to parse metadata JSON: {raw_metadata}")
+
+ if include_score and doc.score is not None:
+ metadata["score"] = doc.score
+
+ # Extract vector — doc.vector() returns list or empty dict
+ raw_vector = doc.vector(_DEFAULT_VECTOR_FIELD)
+ vector = raw_vector if isinstance(raw_vector, list) and len(raw_vector) > 0 else None
+
+ content = doc.field("content") or ""
+
+ return VectorNode(
+ vector_id=str(doc.id),
+ content=str(content),
+ vector=vector,
+ metadata=metadata,
+ )
+
+
+def _apply_filters_post(nodes: list[VectorNode], filters: dict | None) -> list[VectorNode]:
+ """Apply ReMe-style filter dict as post-filtering on metadata.
+
+ Used as a fallback for metadata keys that are NOT promoted to top-level
+ schema columns (and thus cannot be filtered natively by zvec). Promoted
+ fields are handled by zvec's native ``filter`` parameter instead.
+
+ Supports:
+ - Exact match: {"field": value}
+ - Range query: {"field": [start, end]}
+ """
+ if not filters:
+ return nodes
+
+ filtered = []
+ for node in nodes:
+ match = True
+ for key, value in filters.items():
+ if key.startswith("$"):
+ # Skip compound operators for post-filtering
+ continue
+ node_value = node.metadata.get(key)
+
+ # Range query: [start, end]
+ if isinstance(value, list) and len(value) == 2:
+ if node_value is None:
+ match = False
+ break
+ try:
+ if not value[0] <= node_value <= value[1]:
+ match = False
+ break
+ except TypeError:
+ match = False
+ break
+ else:
+ # Exact match
+ if node_value != value:
+ match = False
+ break
+
+ if match:
+ filtered.append(node)
+
+ return filtered
+
+
+class ZvecVectorStore(BaseVectorStore):
+ """Zvec-based vector store implementation.
+
+ Zvec is a high-performance vector database. This adapter bridges the
+ ReMe ``BaseVectorStore`` interface with zvec's Python API.
+
+ Supports local persistent storage via ``db_path``.
+
+ Args:
+ collection_name: Name of the vector collection.
+ db_path: Local storage path for persistent mode.
+ embedding_model: Model used for generating vector embeddings.
+ dimension: Dimensionality of the embedding vectors (default: 1024).
+ distance: Distance metric — cosine / l2 / ip (default: cosine).
+ **kwargs: Additional zvec-specific configuration.
+ """
+
+ def __init__(
+ self,
+ collection_name: str,
+ db_path: str | Path,
+ embedding_model: BaseEmbeddingModel,
+ dimension: int = 1024,
+ distance: str = "cosine",
+ **kwargs: Any,
+ ):
+ """Initialize the Zvec vector store."""
+ if _ZVEC_IMPORT_ERROR is not None:
+ raise ImportError(
+ "Zvec requires extra dependencies. Install with `pip install zvec`",
+ ) from _ZVEC_IMPORT_ERROR
+
+ super().__init__(
+ collection_name=collection_name,
+ db_path=db_path,
+ embedding_model=embedding_model,
+ **kwargs,
+ )
+
+ self.dimension = dimension
+ self.distance = distance
+ self._collection = None
+ self._initialized = False
+ # Set of promoted field names that exist in the current collection's schema.
+ # Populated during start() by inspecting the schema. Only fields present
+ # in the schema can use native zvec filtering; the rest fall back to
+ # Python post-filtering.
+ self._promoted_fields_in_schema: set[str] = set()
+
+ # ------------------------------------------------------------------
+ # Lifecycle
+ # ------------------------------------------------------------------
+
+ async def start(self) -> None:
+ """Initialize the Zvec engine and open the collection.
+
+ Calls ``zvec.init()`` once, then tries to ``zvec.open()`` an existing
+ collection or ``zvec.create_and_open()`` a new one.
+ After opening, detects which promoted fields exist in the schema
+ and attempts to add missing numeric fields via ``add_column``.
+ """
+ if not self._initialized:
+ try:
+ zvec.init()
+ except RuntimeError:
+ # Already initialized — safe to ignore
+ pass
+ self._initialized = True
+
+ self.db_path.mkdir(parents=True, exist_ok=True)
+ collection_path = str(self.db_path / self.collection_name)
+
+ option = CollectionOption(read_only=False, enable_mmap=True)
+
+ try:
+ # Try opening an existing collection first
+ self._collection = zvec.open(collection_path, option)
+ logger.info(f"Opened existing Zvec collection at {collection_path}")
+ except Exception:
+ # Collection doesn't exist — create it
+ schema = _build_collection_schema(
+ name=self.collection_name,
+ dimension=self.dimension,
+ metric=self.distance,
+ )
+ self._collection = zvec.create_and_open(
+ path=collection_path,
+ schema=schema,
+ option=option,
+ )
+ logger.info(f"Created new Zvec collection at {collection_path}")
+
+ # Detect which promoted fields exist in the current schema
+ self._detect_promoted_fields()
+
+ # Try to add missing numeric promoted fields to existing collections
+ # (zvec's add_column only supports numeric types: INT64, FLOAT, etc.)
+ self._ensure_numeric_promoted_columns()
+
+ async def close(self) -> None:
+ """Flush pending writes and release the collection handle."""
+ if self._collection is not None:
+ try:
+ self._collection.flush()
+ except Exception as e:
+ logger.warning(f"Failed to flush collection on close: {e}")
+ self._collection = None
+ logger.info(f"Zvec vector store for collection {self.collection_name} closed")
+
+ # ------------------------------------------------------------------
+ # Collection management
+ # ------------------------------------------------------------------
+
+ async def list_collections(self) -> list[str]:
+ """Retrieve a list of collection names in the db_path directory.
+
+ Zvec doesn't have a global ``list_collections`` API; we scan the
+ db_path directory for zvec collection folders.
+ """
+ if not self.db_path.exists():
+ return []
+ collections = []
+ for child in self.db_path.iterdir():
+ if child.is_dir():
+ collections.append(child.name)
+ return collections
+
+ async def create_collection(self, collection_name: str, **kwargs) -> None:
+ """Create a new collection with the specified name and distance metric."""
+ if not self._initialized:
+ try:
+ zvec.init()
+ except RuntimeError:
+ pass
+ self._initialized = True
+
+ self.db_path.mkdir(parents=True, exist_ok=True)
+ collection_path = str(self.db_path / collection_name)
+
+ dimension = kwargs.get("dimension", self.dimension)
+ metric = kwargs.get("distance_metric", self.distance)
+
+ schema = _build_collection_schema(
+ name=collection_name,
+ dimension=dimension,
+ metric=metric,
+ )
+ option = CollectionOption(read_only=False, enable_mmap=True)
+
+ collection = zvec.create_and_open(path=collection_path, schema=schema, option=option)
+ if collection_name == self.collection_name:
+ self._collection = collection
+ logger.info(f"Created collection `{collection_name}`")
+
+ async def delete_collection(self, collection_name: str, **kwargs) -> None:
+ """Permanently remove a collection from disk."""
+ # If it's the active collection, destroy it via zvec API
+ if self._collection is not None and collection_name == self.collection_name:
+ try:
+ self._collection.destroy()
+ self._collection = None
+ deleted = True
+ except Exception as _e:
+ logger.warning(f"Failed to destroy collection {collection_name}: {_e}")
+ deleted = False
+ else:
+ # For non-active collections, remove the directory
+ collection_path = self.db_path / collection_name
+ if collection_path.exists():
+ import shutil
+
+ shutil.rmtree(collection_path, ignore_errors=True)
+ deleted = True
+ else:
+ deleted = False
+
+ logger.info(f"Deleted collection {collection_name}: {deleted}")
+
+ async def copy_collection(self, collection_name: str, **kwargs) -> None:
+ """Duplicate the current collection to a new one with the given name.
+
+ Uses ``shutil.copytree`` to directly copy the collection directory on
+ disk, which is both faster and complete — it avoids the topk limit of
+ ``list()`` (max 1024 docs) that would cause data loss for large
+ collections.
+
+ The source collection is flushed before copying to ensure all
+ pending writes are persisted to disk.
+ """
+ import shutil
+
+ # Flush source collection so all data is on disk
+ if self._collection is not None:
+ self._collection.flush()
+
+ src_path = self.db_path / self.collection_name
+ dst_path = self.db_path / collection_name
+
+ if not src_path.exists():
+ logger.warning(f"Source collection directory not found: {src_path}")
+ return
+
+ if dst_path.exists():
+ logger.warning(f"Target collection already exists: {dst_path}, removing it first")
+ shutil.rmtree(dst_path, ignore_errors=True)
+
+ shutil.copytree(src_path, dst_path)
+ logger.info(
+ f"Copied collection {self.collection_name} to {collection_name} "
+ f"(directory copy: {src_path} -> {dst_path})",
+ )
+
+ # ------------------------------------------------------------------
+ # CRUD operations
+ # ------------------------------------------------------------------
+
+ async def insert(self, nodes: VectorNode | list[VectorNode], **kwargs) -> None:
+ """Add one or more vector nodes into the current collection.
+
+ Automatically generates embeddings for nodes that lack vectors.
+ """
+ if isinstance(nodes, VectorNode):
+ nodes = [nodes]
+ if not nodes:
+ return
+
+ # Batch generate embeddings for nodes that need them
+ nodes_without_vectors = [n for n in nodes if n.vector is None]
+ if nodes_without_vectors:
+ nodes_with_vectors = await self.get_node_embeddings(nodes_without_vectors)
+ vector_map = {n.vector_id: n for n in nodes_with_vectors}
+ nodes_to_insert = [vector_map.get(n.vector_id, n) if n.vector is None else n for n in nodes]
+ else:
+ nodes_to_insert = nodes
+
+ batch_size = kwargs.get("batch_size", 100)
+
+ for i in range(0, len(nodes_to_insert), batch_size):
+ batch = nodes_to_insert[i : i + batch_size]
+ docs = [_vector_node_to_doc(n) for n in batch]
+ self._collection.insert(docs)
+
+ logger.info(f"Inserted {len(nodes_to_insert)} nodes into {self.collection_name}")
+
+ async def search(
+ self,
+ query: str,
+ limit: int = 5,
+ filters: dict | None = None,
+ **kwargs,
+ ) -> list[VectorNode]:
+ """Find the most similar vector nodes based on a text query.
+
+ Uses zvec's ``query()`` method with a ``VectorQuery`` built from the
+ embedding of the query text. Promoted metadata fields are filtered
+ natively via zvec's ``filter`` parameter; remaining filters are
+ applied as post-filtering in Python.
+ """
+ query_vector = await self.get_embedding(query)
+
+ vq = VectorQuery(
+ field_name=_DEFAULT_VECTOR_FIELD,
+ vector=query_vector,
+ )
+ include_vector = kwargs.get("include_embeddings", False)
+
+ # Split filters: native zvec filter vs Python post-filter
+ native_filter, post_filters = _build_zvec_filter(filters, self._promoted_fields_in_schema)
+
+ # Over-fetch to compensate for post-filtering
+ _ZVEC_MAX_TOPK = 1024
+ # When post-filters remain, we need to fetch more results because
+ # many may be filtered out. Use the maximum allowed to minimize misses.
+ fetch_limit = _ZVEC_MAX_TOPK if post_filters else min(limit, _ZVEC_MAX_TOPK)
+
+ results = self._collection.query(
+ vectors=vq,
+ topk=fetch_limit,
+ filter=native_filter,
+ include_vector=include_vector,
+ )
+
+ nodes = [_doc_to_vector_node(doc, include_score=True) for doc in results]
+
+ # Post-filter on non-promoted metadata fields
+ nodes = _apply_filters_post(nodes, post_filters)
+
+ score_threshold = kwargs.get("score_threshold")
+ if score_threshold is not None:
+ nodes = [n for n in nodes if n.metadata.get("score", 0) >= score_threshold]
+
+ return nodes[:limit]
+
+ async def delete(self, vector_ids: str | list[str], **kwargs) -> None:
+ """Remove specific vectors from the collection using their identifiers."""
+ if isinstance(vector_ids, str):
+ vector_ids = [vector_ids]
+ if not vector_ids:
+ return
+
+ self._collection.delete(vector_ids)
+ logger.info(f"Deleted {len(vector_ids)} nodes from {self.collection_name}")
+
+ async def delete_all(self, **kwargs) -> None:
+ """Remove all vectors from the collection.
+
+ Uses zvec's ``delete_by_filter`` with a condition that matches all
+ documents (content is not empty), or falls back to query + delete
+ in batches (zvec topk max is 1024).
+ """
+ stats = self._collection.stats
+ count = stats.doc_count if stats else 0
+ if count > 0:
+ try:
+ # Use delete_by_filter for efficiency
+ self._collection.delete_by_filter("content!=''")
+ except Exception:
+ # Fallback: fetch all IDs in batches then delete
+ _ZVEC_MAX_TOPK = 1024
+ remaining = count
+ while remaining > 0:
+ all_docs = self._collection.query(topk=min(remaining, _ZVEC_MAX_TOPK), include_vector=False)
+ if not all_docs:
+ break
+ ids = [doc.id for doc in all_docs]
+ self._collection.delete(ids)
+ remaining -= len(ids)
+ logger.info(f"Deleted all {count} nodes from {self.collection_name}")
+
+ async def update(self, nodes: VectorNode | list[VectorNode], **kwargs) -> None:
+ """Update existing vectors using zvec's ``upsert``.
+
+ Automatically regenerates embeddings for nodes whose content changed
+ but lack an updated vector.
+ """
+ if isinstance(nodes, VectorNode):
+ nodes = [nodes]
+ if not nodes:
+ return
+
+ # Batch generate embeddings for nodes that need them
+ nodes_without_vectors = [n for n in nodes if n.vector is None and n.content]
+ if nodes_without_vectors:
+ nodes_with_vectors = await self.get_node_embeddings(nodes_without_vectors)
+ vector_map = {n.vector_id: n for n in nodes_with_vectors}
+ nodes_to_update = [vector_map.get(n.vector_id, n) if n.vector is None and n.content else n for n in nodes]
+ else:
+ nodes_to_update = nodes
+
+ docs = [_vector_node_to_doc(n) for n in nodes_to_update]
+ self._collection.upsert(docs)
+ logger.info(f"Updated {len(nodes_to_update)} nodes in {self.collection_name}")
+
+ async def get(self, vector_ids: str | list[str]) -> VectorNode | list[VectorNode]:
+ """Fetch specific vector nodes from the collection by their IDs."""
+ is_single = isinstance(vector_ids, str)
+ ids = [vector_ids] if is_single else vector_ids
+
+ result_dict = self._collection.fetch(ids)
+ nodes = [_doc_to_vector_node(doc) for doc in result_dict.values()]
+ return nodes[0] if is_single and nodes else (nodes if not is_single else None)
+
+ async def list(
+ self,
+ filters: dict | None = None,
+ limit: int | None = None,
+ sort_key: str | None = None,
+ reverse: bool = True,
+ ) -> list[VectorNode]:
+ """Retrieve vectors matching optional metadata filters.
+
+ Uses zvec's ``query()`` without a vector query to list all documents.
+ Promoted metadata fields are filtered natively via zvec's ``filter``
+ parameter; remaining filters are applied as post-filtering in Python.
+
+ Args:
+ filters: Dictionary of filter conditions to match vectors.
+ limit: Maximum number of vectors to return.
+ sort_key: Key to sort the results by (in metadata).
+ reverse: If True, sort in descending order; otherwise ascending.
+ """
+ # Split filters: native zvec filter vs Python post-filter
+ native_filter, post_filters = _build_zvec_filter(filters, self._promoted_fields_in_schema)
+
+ # Determine fetch limit — zvec max topk is 1024 (will be lifted to 100,000 in zvec v0.3.2+)
+ _ZVEC_MAX_TOPK = 1024
+ fetch_limit = min(limit or _ZVEC_MAX_TOPK, _ZVEC_MAX_TOPK)
+ if sort_key or post_filters:
+ fetch_limit = _ZVEC_MAX_TOPK # fetch max and sort/filter in Python
+
+ results = self._collection.query(
+ topk=fetch_limit,
+ filter=native_filter,
+ include_vector=True,
+ )
+
+ nodes = [_doc_to_vector_node(doc) for doc in results]
+
+ # Post-filter on non-promoted metadata fields
+ nodes = _apply_filters_post(nodes, post_filters)
+
+ # Apply sorting if sort_key is provided
+ if sort_key:
+
+ def _sort_key_func(node: VectorNode):
+ value = node.metadata.get(sort_key)
+ if value is None:
+ return float("-inf") if not reverse else float("inf")
+ return value
+
+ nodes.sort(key=_sort_key_func, reverse=reverse)
+
+ if limit is not None:
+ nodes = nodes[:limit]
+
+ return nodes
+
+ # ------------------------------------------------------------------
+ # Helpers
+ # ------------------------------------------------------------------
+
+ def _detect_promoted_fields(self) -> None:
+ """Detect which promoted fields exist in the current collection's schema.
+
+ Compares the set of promoted field names against the actual schema
+ and populates ``_promoted_fields_in_schema`` accordingly. Only fields
+ present in the schema can use native zvec filtering.
+ """
+ if self._collection is None:
+ return
+
+ try:
+ schema = self._collection.schema
+ existing_fields = {f.name for f in schema.fields} if schema.fields else set()
+ except Exception as e:
+ logger.warning(f"Failed to read collection schema: {e}")
+ existing_fields = set()
+
+ self._promoted_fields_in_schema = set(_PROMOTED_FIELD_SPECS.keys()) & existing_fields
+
+ missing = set(_PROMOTED_FIELD_SPECS.keys()) - existing_fields
+ if missing:
+ logger.info(
+ f"Promoted fields not in schema (will use post-filtering): {missing}",
+ )
+
+ def _ensure_numeric_promoted_columns(self) -> None:
+ """Add missing numeric promoted fields to existing collections.
+
+ zvec's ``add_column`` only supports numeric types (INT64, FLOAT, etc.).
+ STRING fields cannot be added via ``add_column`` and must be defined
+ at collection creation time. For those, we fall back to post-filtering.
+ """
+ if self._collection is None:
+ return
+
+ missing = set(_PROMOTED_FIELD_SPECS.keys()) - self._promoted_fields_in_schema
+ if not missing:
+ return
+
+ # Populate the DataType map if needed
+ if not _DATATYPE_MAP:
+ _DATATYPE_MAP.update(
+ {
+ "STRING": DataType.STRING,
+ "INT64": DataType.INT64,
+ },
+ )
+
+ for field_name in missing:
+ type_str, _ = _PROMOTED_FIELD_SPECS[field_name]
+ # Only numeric types can be added via add_column
+ if type_str not in ("INT64", "INT32", "FLOAT", "DOUBLE"):
+ continue
+ try:
+ dt = _DATATYPE_MAP[type_str]
+ self._collection.add_column(FieldSchema(field_name, dt, nullable=True))
+ self._promoted_fields_in_schema.add(field_name)
+ logger.info(f"Added promoted column '{field_name}' to existing collection")
+ except Exception as e:
+ logger.warning(f"Failed to add column '{field_name}': {e}")
+
+ async def count(self) -> int:
+ """Return the total number of documents in the current collection."""
+ stats = self._collection.stats
+ return stats.doc_count if stats else 0
+
+ async def reset(self):
+ """Reset the current collection by destroying and recreating it."""
+ logger.warning(f"Resetting collection {self.collection_name}...")
+ await self.delete_collection(self.collection_name)
+ await self.create_collection(self.collection_name)
+ logger.info(f"Collection {self.collection_name} has been reset")
diff --git a/tests/test_file_store.py b/tests/test_file_store.py
index fd44332d..0e34764f 100644
--- a/tests/test_file_store.py
+++ b/tests/test_file_store.py
@@ -28,6 +28,7 @@ from reme.core.file_store.base_file_store import BaseFileStore
from reme.core.file_store.chroma_file_store import ChromaFileStore
from reme.core.file_store.local_file_store import LocalFileStore
from reme.core.file_store.sqlite_file_store import SqliteFileStore
+from reme.core.file_store.zvec_file_store import ZvecFileStore
from reme.core.schema.file_metadata import FileMetadata
from reme.core.schema.memory_chunk import MemoryChunk
from reme.core.utils import load_env
@@ -53,6 +54,10 @@ class TestConfig:
CHROMA_DB_PATH = "./test_file_store_chroma"
CHROMA_FTS_ENABLED = True
+ # ZvecFileStore settings
+ ZVEC_DB_PATH = "./test_file_store_zvec"
+ ZVEC_FTS_ENABLED = True
+
# LocalFileStore settings
LOCAL_DB_PATH = "./test_file_store_local"
LOCAL_FTS_ENABLED = True
@@ -199,6 +204,8 @@ def get_store_type(store: BaseFileStore) -> str:
return "chroma"
elif isinstance(store, LocalFileStore):
return "local"
+ elif isinstance(store, ZvecFileStore):
+ return "zvec"
else:
raise ValueError(f"Unknown file store type: {type(store)}")
@@ -242,6 +249,14 @@ def create_file_store(store_type: str) -> BaseFileStore:
embedding_model=embedding_model,
fts_enabled=config.LOCAL_FTS_ENABLED,
)
+ elif store_type == "zvec":
+ return ZvecFileStore(
+ store_name=config.NAME,
+ db_path=config.ZVEC_DB_PATH,
+ embedding_model=embedding_model,
+ fts_enabled=config.ZVEC_FTS_ENABLED,
+ dimension=config.EMBEDDING_DIMENSIONS,
+ )
else:
raise ValueError(f"Unknown store type: {store_type}")
@@ -284,6 +299,13 @@ async def test_start_store(store: BaseFileStore, _store_name: str):
assert isinstance(store._files, dict), "Files index should be a dict"
logger.info(f"✓ LocalFileStore ready (chunks file: {store._chunks_file})")
+ # Verify ZvecFileStore initialized
+ if isinstance(store, ZvecFileStore):
+ # pylint: disable=protected-access
+ assert store._collection is not None, "Zvec collection should be initialized"
+ assert store._initialized, "Zvec engine should be initialized"
+ logger.info(f"✓ ZvecFileStore ready (collection: {store.collection_name})")
+
async def test_upsert_file(store: BaseFileStore, _store_name: str) -> tuple[FileMetadata, List[MemoryChunk]]:
"""Test file and chunks insertion."""
@@ -1013,6 +1035,18 @@ async def cleanup_store(store: BaseFileStore, store_type: str):
json_file.unlink()
logger.info(f"✓ Cleaned up file: {json_file}")
+ # Clean up zvec directory and metadata file
+ if store_type == "zvec":
+ config = TestConfig()
+ db_dir = Path(config.ZVEC_DB_PATH)
+ if db_dir.exists():
+ shutil.rmtree(db_dir)
+ logger.info(f"✓ Cleaned up directory: {db_dir}")
+ metadata_file = db_dir.parent / f"{config.NAME}_file_metadata.json"
+ if metadata_file.exists():
+ metadata_file.unlink()
+ logger.info(f"✓ Cleaned up metadata file: {metadata_file}")
+
logger.info("✓ Cleanup completed")
except Exception as e:
logger.error(f"Cleanup error: {e}")
@@ -1049,6 +1083,11 @@ Examples:
action="store_true",
help="Test LocalFileStore",
)
+ parser.add_argument(
+ "--zvec",
+ action="store_true",
+ help="Test ZvecFileStore",
+ )
parser.add_argument(
"--all",
action="store_true",
@@ -1065,6 +1104,7 @@ Examples:
("sqlite", "SqliteFileStore"),
("chroma", "ChromaFileStore"),
("local", "LocalFileStore"),
+ ("zvec", "ZvecFileStore"),
]
else:
# Build list based on individual flags
@@ -1074,6 +1114,8 @@ Examples:
stores_to_test.append(("chroma", "ChromaFileStore"))
if args.local:
stores_to_test.append(("local", "LocalFileStore"))
+ if args.zvec:
+ stores_to_test.append(("zvec", "ZvecFileStore"))
if not stores_to_test:
# Default to all file stores if no argument provided
@@ -1081,9 +1123,10 @@ Examples:
("sqlite", "SqliteFileStore"),
("chroma", "ChromaFileStore"),
("local", "LocalFileStore"),
+ ("zvec", "ZvecFileStore"),
]
print("No file store specified, defaulting to test all file stores")
- print("Use --sqlite, --chroma, or --local to test specific ones\n")
+ print("Use --sqlite, --chroma, --local, or --zvec to test specific ones\n")
# Run tests for each file store
for store_type, store_name in stores_to_test:
diff --git a/tests/test_vector_store.py b/tests/test_vector_store.py
index 7f7264be..9b39ab9b 100644
--- a/tests/test_vector_store.py
+++ b/tests/test_vector_store.py
@@ -2,7 +2,7 @@
"""Unified test suite for vector store implementations.
This module provides comprehensive test coverage for LocalVectorStore, ESVectorStore,
-PGVectorStore, QdrantVectorStore, ChromaVectorStore, and ObVecVectorStore implementations.
+PGVectorStore, QdrantVectorStore, ChromaVectorStore, ObVecVectorStore and ZvecVectorStore implementations.
Tests can be run for specific vector stores or all implementations.
Usage:
@@ -12,6 +12,7 @@ Usage:
python test_vector_store.py --qdrant # Test QdrantVectorStore only
python test_vector_store.py --chroma # Test ChromaVectorStore only
python test_vector_store.py --obvec # Test ObVecVectorStore only (needs seekdb / OceanBase)
+ python test_vector_store.py --zvec # Test ZvecVectorStore only
python test_vector_store.py --all # Test all vector stores
"""
@@ -36,6 +37,7 @@ from reme.core.vector_store import (
ObVecVectorStore,
PGVectorStore,
QdrantVectorStore,
+ ZvecVectorStore,
)
load_env()
@@ -90,6 +92,9 @@ class TestConfig:
OBVEC_PASSWORD = os.environ.get("OBVEC_PASSWORD", "root")
OBVEC_DATABASE = os.environ.get("OBVEC_DATABASE", "test")
+ # ZvecVectorStore settings
+ ZVEC_PATH = "./test_vector_store_zvec" # For local persistent mode
+
# Embedding model settings
EMBEDDING_MODEL_NAME = "text-embedding-v4"
EMBEDDING_DIMENSIONS = 64
@@ -192,6 +197,7 @@ class SampleDataGenerator:
# ==================== Vector Store Factory ====================
+# pylint: disable=too-many-return-statements
def get_store_type(store: BaseVectorStore) -> str:
"""Get the type identifier of a vector store instance.
@@ -199,7 +205,7 @@ def get_store_type(store: BaseVectorStore) -> str:
store: Vector store instance
Returns:
- str: Type identifier ("local", "es", "pgvector", "qdrant", "chroma", or "obvec")
+ str: Type identifier ("local", "es", "pgvector", "qdrant", "chroma", "obvec", or "zvec")
"""
if isinstance(store, LocalVectorStore):
return "local"
@@ -213,10 +219,13 @@ def get_store_type(store: BaseVectorStore) -> str:
return "chroma"
elif isinstance(store, ObVecVectorStore):
return "obvec"
+ elif isinstance(store, ZvecVectorStore):
+ return "zvec"
else:
raise ValueError(f"Unknown vector store type: {type(store)}")
+# pylint: disable=too-many-return-statements
def create_vector_store(store_type: str, collection_name: str) -> BaseVectorStore:
"""Create a vector store instance based on type.
@@ -295,6 +304,14 @@ def create_vector_store(store_type: str, collection_name: str) -> BaseVectorStor
index_metric="cosine",
index_ef_search=100,
)
+ elif store_type == "zvec":
+ return ZvecVectorStore(
+ collection_name=collection_name,
+ embedding_model=embedding_model,
+ db_path=config.ZVEC_PATH or tempfile.mkdtemp(prefix="test_zvec_"),
+ dimension=config.EMBEDDING_DIMENSIONS,
+ distance="cosine",
+ )
else:
raise ValueError(f"Unknown store type: {store_type}")
@@ -1790,6 +1807,13 @@ async def cleanup_store(store: BaseVectorStore, store_type: str):
shutil.rmtree(obvec_dir, ignore_errors=True)
logger.info(f"Cleaned up obvec temp directory: {obvec_dir}")
+ # Clean up local directory if ZvecVectorStore
+ if store_type == "zvec" and config.ZVEC_PATH:
+ test_dir = Path(config.ZVEC_PATH)
+ if test_dir.exists():
+ shutil.rmtree(test_dir)
+ logger.info(f"Cleaned up zvec directory: {config.ZVEC_PATH}")
+
logger.info("✓ Cleanup completed")
except Exception as e:
logger.error(f"Cleanup error: {e}")
@@ -1844,6 +1868,11 @@ Examples:
action="store_true",
help="Test ObVecVectorStore",
)
+ parser.add_argument(
+ "--zvec",
+ action="store_true",
+ help="Test ZvecVectorStore",
+ )
parser.add_argument(
"--all",
action="store_true",
@@ -1863,6 +1892,7 @@ Examples:
("qdrant", "QdrantVectorStore"),
("chroma", "ChromaVectorStore"),
("obvec", "ObVecVectorStore"),
+ ("zvec", "ZvecVectorStore"),
]
else:
# Build list based on individual flags
@@ -1878,6 +1908,8 @@ Examples:
stores_to_test.append(("chroma", "ChromaVectorStore"))
if args.obvec:
stores_to_test.append(("obvec", "ObVecVectorStore"))
+ if args.zvec:
+ stores_to_test.append(("zvec", "ZvecVectorStore"))
if not stores_to_test:
# Default to all vector stores if no argument provided
@@ -1888,10 +1920,11 @@ Examples:
("qdrant", "QdrantVectorStore"),
("chroma", "ChromaVectorStore"),
("obvec", "ObVecVectorStore"),
+ ("zvec", "ZvecVectorStore"),
]
print("No vector store specified, defaulting to test all vector stores")
print(
- "Use --local/--es/--pgvector/--qdrant/--chroma/--obvec to test specific ones\n",
+ "Use --local/--es/--pgvector/--qdrant/--chroma/--obvec/--zvec to test specific ones\n",
)
# Run tests for each vector store
diff --git a/tests/test_zvec_vector_store.py b/tests/test_zvec_vector_store.py
new file mode 100644
index 00000000..59836f24
--- /dev/null
+++ b/tests/test_zvec_vector_store.py
@@ -0,0 +1,906 @@
+"""Test suite for ZvecVectorStore implementation.
+
+Comprehensive tests covering CRUD operations, search, filtering,
+collection management, and edge cases for the zvec vector store adapter.
+
+Usage:
+ python -m pytest tests/test_zvec_vector_store.py -v
+ python tests/test_zvec_vector_store.py
+"""
+
+# pylint: disable=redefined-outer-name,unused-argument
+
+from __future__ import annotations
+
+import asyncio
+import shutil
+import tempfile
+from pathlib import Path
+from typing import List
+from uuid import uuid4
+
+import pytest
+
+from loguru import logger
+
+from reme.core.schema import VectorNode
+from reme.core.vector_store import ZvecVectorStore
+
+# ---------------------------------------------------------------------------
+# Skip entire module if zvec native library is not installed
+# ---------------------------------------------------------------------------
+try:
+ import zvec as _zvec # noqa: F401 — just checking availability
+except ImportError:
+ pytest.skip("zvec native library not installed", allow_module_location=True)
+
+
+# ==================== Configuration ====================
+
+
+class TestConfig:
+ """Configuration for zvec test execution."""
+
+ ZVEC_ROOT_PATH = tempfile.mkdtemp(prefix="test_zvec_")
+ EMBEDDING_DIMENSION = 64 # Small dimension for faster tests
+ TEST_COLLECTION_PREFIX = "test_zvec_vs"
+
+
+# ==================== Sample Data ====================
+
+
+def create_sample_nodes(prefix: str = "") -> List[VectorNode]:
+ """Create sample VectorNode instances for testing."""
+ id_prefix = f"{prefix}_" if prefix else ""
+ return [
+ VectorNode(
+ vector_id=f"{id_prefix}node1",
+ content="Artificial intelligence is a technology that simulates human intelligence.",
+ metadata={
+ "node_type": "tech",
+ "category": "AI",
+ "source": "research",
+ "priority": "high",
+ "year": "2023",
+ },
+ ),
+ VectorNode(
+ vector_id=f"{id_prefix}node2",
+ content="Machine learning is a subset of artificial intelligence.",
+ metadata={
+ "node_type": "tech",
+ "category": "ML",
+ "source": "research",
+ "priority": "high",
+ "year": "2022",
+ },
+ ),
+ VectorNode(
+ vector_id=f"{id_prefix}node3",
+ content="Deep learning uses neural networks with multiple layers.",
+ metadata={
+ "node_type": "tech_new",
+ "category": "DL",
+ "source": "blog",
+ "priority": "medium",
+ "year": "2024",
+ },
+ ),
+ VectorNode(
+ vector_id=f"{id_prefix}node4",
+ content="I love eating delicious seafood, especially fresh fish.",
+ metadata={
+ "node_type": "food",
+ "category": "preference",
+ "source": "personal",
+ "priority": "low",
+ "year": "2023",
+ },
+ ),
+ VectorNode(
+ vector_id=f"{id_prefix}node5",
+ content="Natural language processing enables computers to understand human language.",
+ metadata={
+ "node_type": "tech",
+ "category": "NLP",
+ "source": "research",
+ "priority": "high",
+ "year": "2024",
+ },
+ ),
+ ]
+
+
+# ==================== Fixtures ====================
+
+
+class MockEmbeddingModel:
+ """A mock embedding model that generates deterministic random vectors.
+
+ Avoids external API calls during testing. Produces unit-normalized
+ vectors so that cosine similarity works correctly.
+ """
+
+ def __init__(self, dimension: int = 64):
+ self.dimension = dimension
+
+ async def get_embedding(self, query: str) -> list[float]:
+ """Generate a deterministic embedding from a query string."""
+ import hashlib
+ import struct
+
+ h = hashlib.sha256(query.encode()).digest()
+ # Repeat hash to fill dimension
+ full_hash = b""
+ while len(full_hash) < self.dimension * 4:
+ full_hash += hashlib.sha256(h + full_hash).digest()
+
+ vec = list(struct.unpack(f"<{self.dimension}f", full_hash[: self.dimension * 4]))
+ # Normalize to unit vector
+ norm = sum(x * x for x in vec) ** 0.5
+ if norm > 0:
+ vec = [x / norm for x in vec]
+ return vec
+
+ async def get_embeddings(self, queries: list[str]) -> list[list[float]]:
+ """Generate embeddings for multiple queries."""
+ return [await self.get_embedding(q) for q in queries]
+
+ async def get_node_embedding(self, node: VectorNode) -> VectorNode:
+ """Assign embedding to a single node."""
+ if node.content:
+ node.vector = await self.get_embedding(node.content)
+ return node
+
+ async def get_node_embeddings(self, nodes: list[VectorNode]) -> list[VectorNode]:
+ """Assign embeddings to multiple nodes."""
+ return [await self.get_node_embedding(n) for n in nodes]
+
+
+@pytest.fixture
+def embedding_model():
+ """Provide a MockEmbeddingModel for tests."""
+ return MockEmbeddingModel(dimension=TestConfig.EMBEDDING_DIMENSION)
+
+
+@pytest.fixture
+def zvec_store(embedding_model, tmp_path):
+ """Create and start a ZvecVectorStore for testing.
+
+ Yields the store and cleans up afterwards.
+ """
+ collection_name = f"{TestConfig.TEST_COLLECTION_PREFIX}_{uuid4().hex[:8]}"
+ store = ZvecVectorStore(
+ collection_name=collection_name,
+ db_path=str(tmp_path / "zvec_db"),
+ embedding_model=embedding_model,
+ dimension=TestConfig.EMBEDDING_DIMENSION,
+ distance="cosine",
+ )
+
+ async def _setup():
+ await store.start()
+ return store
+
+ store = asyncio.get_event_loop().run_until_complete(_setup())
+ yield store
+
+ async def _teardown():
+ try:
+ await store.close()
+ except Exception:
+ pass
+ # Clean up temp directory
+ db_path = Path(str(tmp_path / "zvec_db"))
+ if db_path.exists():
+ shutil.rmtree(db_path, ignore_errors=True)
+
+ asyncio.get_event_loop().run_until_complete(_teardown())
+
+
+# ==================== Helper ====================
+
+
+def run(coro):
+ """Run an async coroutine in the current event loop."""
+ return asyncio.get_event_loop().run_until_complete(coro)
+
+
+# ==================== Test: Collection Lifecycle ====================
+
+
+class TestCollectionLifecycle:
+ """Tests for collection creation, listing, deletion, and copy."""
+
+ def test_create_collection(self, zvec_store):
+ """Test that a collection is created during start()."""
+ collections = run(zvec_store.list_collections())
+ assert zvec_store.collection_name in collections
+
+ def test_list_collections(self, zvec_store):
+ """Test listing collections."""
+ collections = run(zvec_store.list_collections())
+ assert isinstance(collections, list)
+ assert len(collections) >= 1
+
+ def test_delete_collection(self, zvec_store, embedding_model, tmp_path):
+ """Test deleting a collection."""
+ # Create a secondary collection
+ coll_name = f"del_test_{uuid4().hex[:8]}"
+ store2 = ZvecVectorStore(
+ collection_name=coll_name,
+ db_path=str(tmp_path / "zvec_db"),
+ embedding_model=embedding_model,
+ dimension=TestConfig.EMBEDDING_DIMENSION,
+ )
+ run(store2.start())
+
+ collections = run(zvec_store.list_collections())
+ assert coll_name in collections
+
+ run(zvec_store.delete_collection(coll_name))
+
+ collections = run(zvec_store.list_collections())
+ assert coll_name not in collections
+
+ def test_copy_collection(self, zvec_store, embedding_model, tmp_path):
+ """Test copying a collection."""
+ # Insert some data first
+ nodes = create_sample_nodes("copy")
+ run(zvec_store.insert(nodes))
+
+ copy_name = f"copy_test_{uuid4().hex[:8]}"
+ run(zvec_store.copy_collection(copy_name))
+
+ # Verify copy exists
+ collections = run(zvec_store.list_collections())
+ assert copy_name in collections
+
+ # Clean up
+ run(zvec_store.delete_collection(copy_name))
+
+
+# ==================== Test: Insert ====================
+
+
+class TestInsert:
+ """Tests for node insertion (single and batch)."""
+
+ def test_insert_single_node(self, zvec_store):
+ """Test inserting a single node."""
+ node = VectorNode(
+ vector_id="single_1",
+ content="This is a single node insertion test",
+ metadata={"test_type": "single_insert"},
+ )
+ run(zvec_store.insert(node))
+
+ result = run(zvec_store.get("single_1"))
+ assert result is not None
+ assert result.vector_id == "single_1"
+ assert "single node" in result.content
+
+ def test_insert_batch_nodes(self, zvec_store):
+ """Test inserting multiple nodes in batch."""
+ nodes = create_sample_nodes("batch")
+ run(zvec_store.insert(nodes))
+
+ all_nodes = run(zvec_store.list(limit=10))
+ assert len(all_nodes) >= len(nodes)
+
+ def test_insert_node_with_vector(self, zvec_store):
+ """Test inserting a node that already has a vector."""
+ node = VectorNode(
+ vector_id="prevec_1",
+ content="Node with pre-computed vector",
+ vector=[0.1] * TestConfig.EMBEDDING_DIMENSION,
+ metadata={"test_type": "pre_vector"},
+ )
+ run(zvec_store.insert(node))
+
+ result = run(zvec_store.get("prevec_1"))
+ assert result is not None
+ assert result.vector is not None
+
+
+# ==================== Test: Search ====================
+
+
+class TestSearch:
+ """Tests for vector similarity search."""
+
+ @pytest.fixture(autouse=True)
+ def _insert_sample_data(self, zvec_store):
+ """Insert sample data before each search test."""
+ nodes = create_sample_nodes("search")
+ run(zvec_store.insert(nodes))
+
+ def test_basic_search(self, zvec_store):
+ """Test basic vector search."""
+ results = run(zvec_store.search(query="What is artificial intelligence?", limit=3))
+ assert len(results) > 0
+ for r in results:
+ assert isinstance(r, VectorNode)
+ assert r.content
+
+ def test_search_with_limit(self, zvec_store):
+ """Test search with various limits."""
+ results = run(zvec_store.search(query="technology", limit=2))
+ assert len(results) <= 2
+
+ def test_search_with_filter(self, zvec_store):
+ """Test vector search with metadata filter."""
+ results = run(
+ zvec_store.search(
+ query="What is artificial intelligence?",
+ limit=5,
+ filters={"node_type": "tech"},
+ ),
+ )
+ # All results should have node_type == "tech"
+ for r in results:
+ assert r.metadata.get("node_type") == "tech"
+
+ def test_search_with_multiple_filters(self, zvec_store):
+ """Test search with multiple metadata filters (AND)."""
+ results = run(
+ zvec_store.search(
+ query="What is artificial intelligence?",
+ limit=5,
+ filters={"node_type": "tech", "source": "research"},
+ ),
+ )
+ for r in results:
+ assert r.metadata.get("node_type") == "tech"
+ assert r.metadata.get("source") == "research"
+
+ def test_search_relevance_ranking(self, zvec_store):
+ """Test that search results have scores and are relevant."""
+ results = run(zvec_store.search(query="artificial intelligence", limit=5))
+ assert len(results) > 0
+ # All results should have a score
+ for r in results:
+ assert "score" in r.metadata
+ assert r.metadata["score"] > 0
+ # The top result should be highly relevant (AI content matches AI query)
+ top_content = results[0].content.lower()
+ assert "artificial intelligence" in top_content or "intelligence" in top_content or "ai" in top_content
+
+
+# ==================== Test: Get ====================
+
+
+class TestGet:
+ """Tests for retrieving nodes by ID."""
+
+ @pytest.fixture(autouse=True)
+ def _insert_sample_data(self, zvec_store):
+ """Insert sample data before each get test."""
+ nodes = create_sample_nodes("get")
+ run(zvec_store.insert(nodes))
+
+ def test_get_single_id(self, zvec_store):
+ """Test retrieving a single node by ID."""
+ result = run(zvec_store.get("get_node1"))
+ assert result is not None
+ assert result.vector_id == "get_node1"
+
+ def test_get_multiple_ids(self, zvec_store):
+ """Test retrieving multiple nodes by IDs."""
+ results = run(zvec_store.get(["get_node1", "get_node2"]))
+ assert isinstance(results, list)
+ assert len(results) >= 2
+ result_ids = {r.vector_id for r in results}
+ assert "get_node1" in result_ids
+ assert "get_node2" in result_ids
+
+ def test_get_nonexistent_id(self, zvec_store):
+ """Test retrieving a non-existent ID."""
+ result = run(zvec_store.get("nonexistent_id_xyz"))
+ assert result is None or result == []
+
+
+# ==================== Test: List ====================
+
+
+class TestList:
+ """Tests for listing nodes with optional filters and sorting."""
+
+ @pytest.fixture(autouse=True)
+ def _insert_sample_data(self, zvec_store):
+ """Insert sample data before each list test."""
+ nodes = create_sample_nodes("list")
+ run(zvec_store.insert(nodes))
+
+ def test_list_all(self, zvec_store):
+ """Test listing all nodes."""
+ results = run(zvec_store.list(limit=20))
+ assert len(results) > 0
+
+ def test_list_with_filter(self, zvec_store):
+ """Test listing nodes with metadata filter."""
+ results = run(zvec_store.list(filters={"category": "AI"}, limit=10))
+ for r in results:
+ assert r.metadata.get("category") == "AI"
+
+ def test_list_with_sorting(self, zvec_store):
+ """Test listing with sorting by metadata key."""
+ # Insert nodes with numeric metadata for sorting
+ sort_nodes = [
+ VectorNode(
+ vector_id=f"sort_{i}",
+ content=f"Sort test node {i}",
+ metadata={"rating": str(50 + i * 5), "test_type": "sort_test"},
+ )
+ for i in range(10)
+ ]
+ run(zvec_store.insert(sort_nodes))
+
+ results = run(
+ zvec_store.list(
+ filters={"test_type": "sort_test"},
+ sort_key="rating",
+ reverse=True,
+ limit=5,
+ ),
+ )
+ assert len(results) <= 5
+ # Verify descending order
+ ratings = [r.metadata.get("rating") for r in results]
+ for i in range(len(ratings) - 1):
+ assert ratings[i] >= ratings[i + 1]
+
+
+# ==================== Test: Update ====================
+
+
+class TestUpdate:
+ """Tests for updating existing nodes."""
+
+ @pytest.fixture(autouse=True)
+ def _insert_sample_data(self, zvec_store):
+ """Insert sample data before each update test."""
+ nodes = create_sample_nodes("upd")
+ run(zvec_store.insert(nodes))
+
+ def test_update_single_node(self, zvec_store):
+ """Test updating a single node's content and metadata."""
+ updated = VectorNode(
+ vector_id="upd_node2",
+ content="Machine learning is a powerful subset of AI that learns from data.",
+ metadata={
+ "node_type": "tech",
+ "category": "ML",
+ "updated": "true",
+ },
+ )
+ run(zvec_store.update(updated))
+
+ result = run(zvec_store.get("upd_node2"))
+ assert result is not None
+ assert result.metadata.get("updated") == "true"
+
+ def test_update_batch(self, zvec_store):
+ """Test batch updating multiple nodes."""
+ updates = [
+ VectorNode(
+ vector_id="upd_node1",
+ content="Updated content for node 1",
+ metadata={"node_type": "tech", "batch_updated": "true"},
+ ),
+ VectorNode(
+ vector_id="upd_node3",
+ content="Updated content for node 3",
+ metadata={"node_type": "tech_new", "batch_updated": "true"},
+ ),
+ ]
+ run(zvec_store.update(updates))
+
+ results = run(zvec_store.get(["upd_node1", "upd_node3"]))
+ if isinstance(results, list):
+ for r in results:
+ assert r.metadata.get("batch_updated") == "true"
+
+
+# ==================== Test: Delete ====================
+
+
+class TestDelete:
+ """Tests for deleting nodes."""
+
+ @pytest.fixture(autouse=True)
+ def _insert_sample_data(self, zvec_store):
+ """Insert sample data before each delete test."""
+ nodes = create_sample_nodes("del")
+ run(zvec_store.insert(nodes))
+
+ def test_delete_single(self, zvec_store):
+ """Test deleting a single node by ID."""
+ run(zvec_store.delete("del_node4"))
+
+ # Verify deletion
+ result = run(zvec_store.get("del_node4"))
+ assert result is None or result == []
+
+ def test_delete_batch(self, zvec_store):
+ """Test batch deleting multiple nodes by IDs."""
+ # First insert some extra nodes to delete
+ extra_nodes = [
+ VectorNode(
+ vector_id=f"del_extra_{i}",
+ content=f"Extra node {i} for batch delete test",
+ metadata={"test_type": "batch_delete"},
+ )
+ for i in range(5)
+ ]
+ run(zvec_store.insert(extra_nodes))
+
+ ids = [f"del_extra_{i}" for i in range(5)]
+ run(zvec_store.delete(ids))
+
+ # Verify all deleted
+ for nid in ids:
+ result = run(zvec_store.get(nid))
+ assert result is None or result == []
+
+ def test_delete_all(self, zvec_store):
+ """Test deleting all nodes from the collection."""
+ run(zvec_store.delete_all())
+ # Collection should be empty now
+ remaining = run(zvec_store.list(limit=100))
+ assert len(remaining) == 0
+
+
+# ==================== Test: Edge Cases ====================
+
+
+class TestEdgeCases:
+ """Tests for edge cases and boundary conditions."""
+
+ def test_empty_content(self, zvec_store):
+ """Test inserting a node with empty content."""
+ node = VectorNode(
+ vector_id="edge_empty",
+ content="",
+ metadata={"type": "empty"},
+ )
+ # Empty content may fail embedding — that's OK, we just want to see it handled
+ try:
+ run(zvec_store.insert([node]))
+ except Exception:
+ pass # Expected if embedding fails on empty string
+
+ def test_long_content(self, zvec_store):
+ """Test inserting a node with very long content."""
+ node = VectorNode(
+ vector_id="edge_long",
+ content="A" * 5000,
+ metadata={"type": "long_content"},
+ )
+ run(zvec_store.insert([node]))
+ result = run(zvec_store.get("edge_long"))
+ assert result is not None
+ assert len(result.content) == 5000
+
+ def test_special_characters(self, zvec_store):
+ """Test content with special characters."""
+ node = VectorNode(
+ vector_id="edge_special",
+ content="Special chars: @#$%^&*()[]{}|;:',.<>?/~`",
+ metadata={"type": "special_chars"},
+ )
+ run(zvec_store.insert([node]))
+ result = run(zvec_store.get("edge_special"))
+ assert result is not None
+ assert "@#$%" in result.content
+
+ def test_unicode_content(self, zvec_store):
+ """Test content with Unicode characters."""
+ node = VectorNode(
+ vector_id="edge_unicode",
+ content="Unicode test: 你好世界 مرحبا Привет",
+ metadata={"type": "unicode"},
+ )
+ run(zvec_store.insert([node]))
+ result = run(zvec_store.get("edge_unicode"))
+ assert result is not None
+ assert "你好世界" in result.content
+
+ def test_nonexistent_id(self, zvec_store):
+ """Test getting a non-existent ID."""
+ result = run(zvec_store.get("nonexistent_xyz_999"))
+ assert result is None or result == []
+
+ def test_metadata_with_empty_string_value(self, zvec_store):
+ """Test metadata containing empty string values."""
+ node = VectorNode(
+ vector_id="edge_meta_empty",
+ content="Testing empty metadata values",
+ metadata={"field1": "value1", "field2": "", "field3": "value3"},
+ )
+ run(zvec_store.insert([node]))
+ result = run(zvec_store.get("edge_meta_empty"))
+ assert result is not None
+
+ def test_search_nonexistent_filter(self, zvec_store):
+ """Test search with a filter value that doesn't match anything."""
+ nodes = create_sample_nodes("edge_filter")
+ run(zvec_store.insert(nodes))
+
+ results = run(
+ zvec_store.search(
+ query="test",
+ limit=10,
+ filters={"category": "NONEXISTENT_CATEGORY"},
+ ),
+ )
+ assert len(results) == 0
+
+
+# ==================== Test: Batch Operations ====================
+
+
+class TestBatchOperations:
+ """Tests for large-scale batch insert, update, and delete."""
+
+ def test_batch_insert_100_nodes(self, zvec_store):
+ """Test inserting 100 nodes in batch."""
+ batch_nodes = [
+ VectorNode(
+ vector_id=f"batch_{i}",
+ content=f"This is batch test content number {i} about technology and science.",
+ metadata={
+ "batch_id": str(i // 10),
+ "index": str(i),
+ "category": ["tech", "science", "business"][i % 3],
+ },
+ )
+ for i in range(100)
+ ]
+ run(zvec_store.insert(batch_nodes))
+
+ all_nodes = run(zvec_store.list(limit=150))
+ assert len(all_nodes) >= 100
+
+ def test_batch_update_20_nodes(self, zvec_store):
+ """Test batch updating 20 nodes."""
+ # Insert first
+ nodes = [
+ VectorNode(
+ vector_id=f"bupd_{i}",
+ content=f"Batch update test {i}",
+ metadata={"index": str(i)},
+ )
+ for i in range(30)
+ ]
+ run(zvec_store.insert(nodes))
+
+ # Update first 20
+ updates = [
+ VectorNode(
+ vector_id=f"bupd_{i}",
+ content=f"UPDATED content {i}",
+ metadata={"index": str(i), "updated": "true"},
+ )
+ for i in range(20)
+ ]
+ run(zvec_store.update(updates))
+
+ # Verify
+ results = run(zvec_store.list(filters={"updated": "true"}, limit=30))
+ assert len(results) >= 20
+
+ def test_batch_delete_50_nodes(self, zvec_store):
+ """Test batch deleting 50 nodes."""
+ # Insert
+ nodes = [
+ VectorNode(
+ vector_id=f"bdel_{i}",
+ content=f"Batch delete test {i}",
+ metadata={"index": str(i)},
+ )
+ for i in range(50)
+ ]
+ run(zvec_store.insert(nodes))
+
+ # Delete
+ ids = [f"bdel_{i}" for i in range(50)]
+ run(zvec_store.delete(ids))
+
+ # Verify
+ remaining = run(zvec_store.list(limit=200))
+ batch_remaining = [n for n in remaining if n.vector_id.startswith("bdel_")]
+ assert len(batch_remaining) == 0
+
+
+# ==================== Test: Concurrent Operations ====================
+
+
+class TestConcurrentOperations:
+ """Tests for concurrent read/write operations."""
+
+ def test_concurrent_inserts_and_searches(self, zvec_store):
+ """Test that concurrent inserts and searches work without errors."""
+
+ async def _run():
+ # Concurrent inserts
+ insert_tasks = []
+ for i in range(5):
+ batch = [
+ VectorNode(
+ vector_id=f"conc_{i}_{j}",
+ content=f"Concurrent test content {i}-{j}",
+ metadata={"thread_id": str(i)},
+ )
+ for j in range(10)
+ ]
+ insert_tasks.append(zvec_store.insert(batch))
+
+ await asyncio.gather(*insert_tasks)
+
+ # Concurrent searches
+ search_tasks = [zvec_store.search(query="concurrent test", limit=5) for _ in range(5)]
+ search_results = await asyncio.gather(*search_tasks)
+
+ # All searches should return results
+ for results in search_results:
+ assert len(results) > 0
+
+ run(_run())
+
+
+# ==================== Test: Data Model Conversion ====================
+
+
+class TestDataModelConversion:
+ """Tests for VectorNode <-> zvec Doc conversion helpers."""
+
+ def test_vector_node_to_doc_roundtrip(self, zvec_store):
+ """Test that VectorNode -> Doc -> VectorNode roundtrip preserves data."""
+ from reme.core.vector_store.zvec_vector_store import (
+ _vector_node_to_doc,
+ _doc_to_vector_node,
+ )
+
+ original = VectorNode(
+ vector_id="roundtrip_1",
+ content="Roundtrip test content",
+ vector=[0.1] * TestConfig.EMBEDDING_DIMENSION,
+ metadata={"key1": "value1", "key2": "42", "key3": "true"},
+ )
+
+ doc = _vector_node_to_doc(original)
+ assert doc.id == "roundtrip_1"
+ assert doc.field("content") == "Roundtrip test content"
+
+ restored = _doc_to_vector_node(doc, include_score=False)
+ assert restored.vector_id == "roundtrip_1"
+ assert restored.content == "Roundtrip test content"
+ assert restored.metadata.get("key1") == "value1"
+
+ def test_post_filter_exact_match(self):
+ """Test post-filtering with exact match."""
+ from reme.core.vector_store.zvec_vector_store import _apply_filters_post
+
+ nodes = [
+ VectorNode(vector_id="1", content="a", metadata={"category": "AI"}),
+ VectorNode(vector_id="2", content="b", metadata={"category": "ML"}),
+ VectorNode(vector_id="3", content="c", metadata={"category": "AI"}),
+ ]
+
+ filtered = _apply_filters_post(nodes, {"category": "AI"})
+ assert len(filtered) == 2
+ assert all(n.metadata["category"] == "AI" for n in filtered)
+
+ def test_post_filter_range_query(self):
+ """Test post-filtering with range query."""
+ from reme.core.vector_store.zvec_vector_store import _apply_filters_post
+
+ nodes = [
+ VectorNode(vector_id="1", content="a", metadata={"year": 2022}),
+ VectorNode(vector_id="2", content="b", metadata={"year": 2023}),
+ VectorNode(vector_id="3", content="c", metadata={"year": 2024}),
+ ]
+
+ filtered = _apply_filters_post(nodes, {"year": [2023, 2024]})
+ assert len(filtered) == 2
+
+ def test_post_filter_none_and_empty(self):
+ """Test post-filtering with None and empty filters."""
+ from reme.core.vector_store.zvec_vector_store import _apply_filters_post
+
+ nodes = [VectorNode(vector_id="1", content="a", metadata={})]
+
+ # None filter returns all
+ assert _apply_filters_post(nodes, None) == nodes
+ # Empty filter returns all
+ assert _apply_filters_post(nodes, {}) == nodes
+
+ def test_score_excluded_from_stored_metadata(self):
+ """Test that score is excluded when converting VectorNode to Doc."""
+ from reme.core.vector_store.zvec_vector_store import _vector_node_to_doc
+
+ node = VectorNode(
+ vector_id="score_test",
+ content="test",
+ vector=[0.1] * TestConfig.EMBEDDING_DIMENSION,
+ metadata={"key1": "val1", "score": 0.95},
+ )
+
+ doc = _vector_node_to_doc(node)
+ # The metadata JSON should NOT contain the score key
+ import json
+
+ stored_meta = json.loads(doc.field("metadata"))
+ assert "score" not in stored_meta
+ assert "key1" in stored_meta
+
+
+# ==================== Main Entry Point ====================
+
+
+async def run_standalone_tests():
+ """Run tests standalone (without pytest) for quick validation."""
+ tmp_dir = tempfile.mkdtemp(prefix="test_zvec_standalone_")
+ embedding_model = MockEmbeddingModel(dimension=TestConfig.EMBEDDING_DIMENSION)
+
+ store = ZvecVectorStore(
+ collection_name="standalone_test",
+ db_path=tmp_dir,
+ embedding_model=embedding_model,
+ dimension=TestConfig.EMBEDDING_DIMENSION,
+ distance="cosine",
+ )
+
+ try:
+ await store.start()
+ logger.info("✓ Store started")
+
+ # Insert
+ nodes = create_sample_nodes("std")
+ await store.insert(nodes)
+ logger.info(f"✓ Inserted {len(nodes)} nodes")
+
+ # Search
+ results = await store.search(query="artificial intelligence", limit=3)
+ logger.info(f"✓ Search returned {len(results)} results")
+ for r in results:
+ logger.info(f" - {r.vector_id}: {r.content[:50]}... (score={r.metadata.get('score')})")
+
+ # Get
+ result = await store.get("std_node1")
+ logger.info(f"✓ Get: {result.vector_id if result else 'None'}")
+
+ # List
+ all_nodes = await store.list(limit=10)
+ logger.info(f"✓ List: {len(all_nodes)} nodes")
+
+ # Update
+ await store.update(
+ VectorNode(
+ vector_id="std_node1",
+ content="Updated content",
+ metadata={"updated": "true"},
+ ),
+ )
+ result = await store.get("std_node1")
+ logger.info(f"✓ Update: metadata.updated={result.metadata.get('updated') if result else 'N/A'}")
+
+ # Delete
+ await store.delete("std_node4")
+ result = await store.get("std_node4")
+ logger.info(f"✓ Delete: {'gone' if result is None or result == [] else 'still exists'}")
+
+ # Count
+ count = await store.count()
+ logger.info(f"✓ Count: {count} nodes")
+
+ logger.info("✓ All standalone tests passed!")
+
+ finally:
+ await store.close()
+ shutil.rmtree(tmp_dir, ignore_errors=True)
+
+
+if __name__ == "__main__":
+ asyncio.run(run_standalone_tests())
diff --git a/tests/vector/test_reme_vector.py b/tests/vector/test_reme_vector.py
index 3829c841..8a37cb97 100644
--- a/tests/vector/test_reme_vector.py
+++ b/tests/vector/test_reme_vector.py
@@ -20,7 +20,7 @@ async def main():
"dimensions": 1024,
},
default_vector_store_config={
- "backend": "local", # 支持 local/chroma/qdrant/elasticsearch
+ "backend": "local", # 支持 local/chroma/qdrant/elasticsearch/zvec
},
)
await reme.start()
From d72f5fc581b1e770f134dedf9e998d6f2aac907f Mon Sep 17 00:00:00 2001
From: yangtiancheng-ali
Date: Sat, 9 May 2026 10:30:37 +0800
Subject: [PATCH 06/16] feat(vector_store): add Hologres vector store
implementation (#226)
---
README.md | 2 +-
docs/index.md | 2 +-
docs/vector_store_api_guide.md | 52 +-
reme/core/vector_store/__init__.py | 3 +
reme/core/vector_store/hologres_store.py | 633 +++++++++++++++++++++++
tests/test_vector_store.py | 56 +-
6 files changed, 737 insertions(+), 11 deletions(-)
create mode 100644 reme/core/vector_store/hologres_store.py
diff --git a/README.md b/README.md
index b4bb48c4..b408332b 100644
--- a/README.md
+++ b/README.md
@@ -506,7 +506,7 @@ async def main():
"dimensions": 1024,
},
default_vector_store_config={
- "backend": "local", # Supports local/chroma/qdrant/elasticsearch/obvec/zvec
+ "backend": "local", # Supports local/chroma/qdrant/elasticsearch/obvec/zvec/hologres
},
)
await reme.start()
diff --git a/docs/index.md b/docs/index.md
index d7e2e716..d0e9d9d2 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -139,7 +139,7 @@ response = requests.post("http://localhost:8002/retrieve_task_memory", json={
## 📚 Resources
- **[Installation Guide](installation.md)**, **[Quick Start](quick_start.md)**: Get started quickly with practical examples
-- **[Vector Storage Setup](vector_store_api_guide.md)**: Configure local, Elasticsearch, Qdrant, ChromaDB, or ObVec (OceanBase / seekdb via pyobvector) storage and usage
+- **[Vector Storage Setup](vector_store_api_guide.md)**: Configure local, Elasticsearch, Qdrant, ChromaDB, ObVec (OceanBase / seekdb via pyobvector) or Hologres storage and usage
- **[MCP Guide](mcp_quick_start.md)**: Create MCP services
- **[Personal Memory](personal_memory/personal_memory.md)**, **[Task Memory](task_memory/task_memory.md)** & **[Tool Memory](tool_memory/tool_memory.md)**: Operators used in personal memory, task memory and tool memory. You can modify the config to customize the pipelines.
- **[Example Collection](./cookbook/appworld/quickstart.md)**: Real use cases and best practices
diff --git a/docs/vector_store_api_guide.md b/docs/vector_store_api_guide.md
index 89881ef2..53ced358 100644
--- a/docs/vector_store_api_guide.md
+++ b/docs/vector_store_api_guide.md
@@ -34,6 +34,7 @@ FlowLLM provides multiple Vector Store implementations tailored to different use
- **ChromaVectorStore** ([source code](https://github.com/flowllm-ai/flowllm/blob/main/flowllm/core/vector_store/chroma_vector_store.py)): Based on ChromaDB, providing persistent storage and metadata filtering capabilities.
- **EsVectorStore** ([source code](https://github.com/flowllm-ai/flowllm/blob/main/flowllm/core/vector_store/es_vector_store.py)): Built on Elasticsearch, enabling powerful combined full-text and vector search functionalities.
- **ObVecVectorStore** ([source code](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/obvec_vector_store.py)): Uses [pyobvector](https://pypi.org/project/pyobvector/) against **OceanBase** or **seekdb** (MySQL-compatible wire protocol). Suitable when you already run OceanBase/seekdb or need a SQL-native vector table with HNSW-style ANN search and JSON metadata filters.
+- **HologresVectorStore** ([source code](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/hologres_store.py)): Uses [asyncpg](https://pypi.org/project/asyncpg/) against **Hologres** (PostgreSQL-compatible). Leverages native `float4[]` vector storage with built-in HGraph index for approximate nearest neighbor search. Suitable when you already run Hologres or need high-performance vector search with JSONB metadata filtering in a PostgreSQL-compatible environment.
- **ZvecVectorStore** ([source code](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/zvec_vector_store.py)): Built on zvec, a high-performance local vector database with strong-schema support and HNSW indexing. Suitable for single-machine deployments requiring fast vector search.
All Vector Store implementations inherit from **BaseVectorStore** ([source code](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/base_vector_store.py)) in ReMe, ensuring a consistent interface specification.
@@ -137,6 +138,20 @@ OBVEC_PASSWORD= python tests/test_vector_store.py --obvec
- **dimension**: Dimensionality of the embedding vectors (default: `1024`).
- **distance**: Distance metric — supports `cosine`, `l2`, `ip` (default: `cosine`).
+### HologresVectorStore Configuration
+
+- **host**: Hologres host address (default: `localhost`).
+- **port**: Hologres port (default: `80`).
+- **database**: Database name (default: `postgres`).
+- **user**: Database user (default: `postgres`).
+- **password**: Database password.
+- **schema**: PostgreSQL schema name (default: `public`).
+- **min_size**: Minimum connections in pool (default: `1`).
+- **max_size**: Maximum connections in pool (default: `10`).
+- **dsn**: Full DSN connection string. When provided, overrides `host`, `port`, `database`, `user`, and `password`.
+- **distance_method**: Distance method for the HGraph index: `Cosine`, `InnerProduct`, or `Euclidean` (default: `Cosine`).
+- **collection_name**: Table name for the collection (from `VectorStoreConfig`, default `reme`).
+
## Configuration File Examples
Configure Vector Store in `flowllm/config/default.yaml` under the `vector_store` section. The basic structure is as follows:
@@ -157,7 +172,7 @@ vector_store.default.params.=
### Configuration Field Descriptions
-- **`backend`** (required): Vector store backend type. Options: `local`, `memory`, `chroma`, `qdrant`, `elasticsearch`, `obvec`, `zvec`.
+- **`backend`** (required): Vector store backend type. Options: `local`, `memory`, `chroma`, `qdrant`, `elasticsearch`, `obvec`, `zvec`, `hologres`.
- **`embedding_model`** (required): Name of the embedding model configuration, referencing the `embedding_model` section.
- **`params`** (optional): Dictionary of backend-specific parameters passed to the vector store constructor.
@@ -353,7 +368,37 @@ vector_stores.default.password=your-root-password
ReMe service YAML uses the key `vector_stores` (plural); CLI overrides use the same nested paths.
-#### 7. ZvecVectorStore Configuration
+#### 7. HologresVectorStore Configuration
+
+**Implementation**: [`reme/core/vector_store/hologres_store.py`](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/hologres_store.py)
+
+**Example (Hologres instance)**:
+
+```yaml
+vector_stores:
+ default:
+ backend: hologres
+ embedding_model: default
+ collection_name: reme
+ host: "your-hologres-host"
+ port: 80
+ database: "postgres"
+ user: "postgres"
+ password: "your-password"
+ schema: "public"
+ distance_method: "Cosine"
+```
+
+```shell
+vector_stores.default.backend=hologres
+vector_stores.default.host=your-hologres-host
+vector_stores.default.port=80
+vector_stores.default.user=postgres
+vector_stores.default.password=your-password
+vector_stores.default.database=postgres
+```
+
+#### 8. ZvecVectorStore Configuration
Persistent local storage based on zvec with HNSW indexing and strong-schema support.
@@ -434,10 +479,11 @@ Two types of metadata filtering are supported:
- **Development & Testing**: Use MemoryVectorStore or LocalVectorStore—no additional services required.
- **Small-Scale Applications**: Use LocalVectorStore or ChromaVectorStore for simplicity and ease of use.
-- **Production Environments**: Use QdrantVectorStore, EsVectorStore, or ObVecVectorStore (OceanBase/seekdb) for high performance and scalability, depending on your existing infrastructure.
+- **Production Environments**: Use QdrantVectorStore, EsVectorStore, ObVecVectorStore (OceanBase/seekdb), or HologresVectorStore for high performance and scalability, depending on your existing infrastructure.
- **High-Performance Local Search**: Use ZvecVectorStore for single-machine deployments requiring fast HNSW-based vector search with local persistence.
- **Hybrid Search**: Use EsVectorStore to combine vector search with full-text search capabilities.
- **OceanBase / seekdb**: Use ObVecVectorStore when you standardize on pyobvector and SQL-accessible vector tables.
+- **Hologres**: Use HologresVectorStore when you run Hologres and need native HGraph-indexed vector search with PostgreSQL-compatible SQL and JSONB metadata filtering.
## Important Notes
diff --git a/reme/core/vector_store/__init__.py b/reme/core/vector_store/__init__.py
index 0426fd64..7b5a60bb 100644
--- a/reme/core/vector_store/__init__.py
+++ b/reme/core/vector_store/__init__.py
@@ -3,6 +3,7 @@
from .base_vector_store import BaseVectorStore
from .chroma_vector_store import ChromaVectorStore
from .es_vector_store import ESVectorStore
+from .hologres_store import HologresVectorStore
from .local_vector_store import LocalVectorStore
from .obvec_vector_store import ObVecVectorStore
from .pgvector_store import PGVectorStore
@@ -14,6 +15,7 @@ __all__ = [
"BaseVectorStore",
"ChromaVectorStore",
"ESVectorStore",
+ "HologresVectorStore",
"LocalVectorStore",
"ObVecVectorStore",
"PGVectorStore",
@@ -23,6 +25,7 @@ __all__ = [
R.vector_stores.register("chroma")(ChromaVectorStore)
R.vector_stores.register("es")(ESVectorStore)
+R.vector_stores.register("hologres")(HologresVectorStore)
R.vector_stores.register("local")(LocalVectorStore)
R.vector_stores.register("obvec")(ObVecVectorStore)
R.vector_stores.register("pgvector")(PGVectorStore)
diff --git a/reme/core/vector_store/hologres_store.py b/reme/core/vector_store/hologres_store.py
new file mode 100644
index 00000000..270dc1bf
--- /dev/null
+++ b/reme/core/vector_store/hologres_store.py
@@ -0,0 +1,633 @@
+"""Hologres implementation for vector storage and retrieval."""
+
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+
+from .base_vector_store import BaseVectorStore
+from ..embedding import BaseEmbeddingModel
+from ..schema import VectorNode
+
+_ASYNCPG_IMPORT_ERROR: Exception | None = None
+
+try:
+ import asyncpg
+ from asyncpg import Pool
+except Exception as e:
+ _ASYNCPG_IMPORT_ERROR = e
+ asyncpg = None
+ Pool = None
+
+
+class HologresVectorStore(BaseVectorStore):
+ """Vector store implementation using Hologres for efficient similarity search.
+
+ Hologres uses native float4[] arrays for vector storage with built-in
+ HGraph index for approximate nearest neighbor search, unlike pgvector
+ which requires an extension.
+ """
+
+ @staticmethod
+ def _validate_table_name(name: str) -> None:
+ """Validate table name to prevent SQL injection."""
+ if not name:
+ raise ValueError("Table name cannot be empty")
+ if len(name) > 63:
+ raise ValueError(f"Table name too long: {len(name)} characters (max 63)")
+ if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", name):
+ raise ValueError(
+ f"Invalid table name: {name}. Must start with letter or underscore, "
+ "and contain only alphanumeric characters and underscores.",
+ )
+
+ def __init__(
+ self,
+ collection_name: str,
+ db_path: str | Path,
+ embedding_model: BaseEmbeddingModel,
+ host: str = "localhost",
+ port: int = 80,
+ database: str = "postgres",
+ user: str = "postgres",
+ password: str = "",
+ schema: str = "public",
+ min_size: int = 1,
+ max_size: int = 10,
+ dsn: str | None = None,
+ distance_method: str = "Cosine",
+ **kwargs,
+ ):
+ """Initialize the Hologres vector store with connection parameters.
+
+ Args:
+ collection_name: Name of the collection (table).
+ db_path: Database path (used by base class).
+ embedding_model: Embedding model for generating vectors.
+ host: Hologres host address.
+ port: Hologres port (default 80 for Hologres).
+ database: Database name.
+ user: Database user.
+ password: Database password.
+ schema: PostgreSQL schema name (default "public").
+ min_size: Minimum connections in pool.
+ max_size: Maximum connections in pool.
+ dsn: Full DSN connection string (overrides individual params).
+ distance_method: Distance method for HGraph index (Cosine, InnerProduct, Euclidean).
+ """
+ if _ASYNCPG_IMPORT_ERROR is not None:
+ raise ImportError(
+ "Hologres vector store requires asyncpg. Install with `pip install asyncpg`",
+ ) from _ASYNCPG_IMPORT_ERROR
+
+ self._validate_table_name(collection_name)
+ self._validate_table_name(schema)
+
+ super().__init__(
+ collection_name=collection_name,
+ db_path=db_path,
+ embedding_model=embedding_model,
+ **kwargs,
+ )
+
+ self.dsn = dsn
+ self.host = host
+ self.port = port
+ self.database = database
+ self.user = user
+ self.password = password
+ self.schema = schema
+ self.min_size = min_size
+ self.max_size = max_size
+ self.distance_method = distance_method
+ self._pool: Pool | None = None
+ self.embedding_model_dims = embedding_model.dimensions
+
+ @property
+ def _qualified_name(self) -> str:
+ """Return the schema-qualified table name (e.g. 'my_schema.my_table')."""
+ return f"{self.schema}.{self.collection_name}"
+
+ def _qualify(self, table_name: str) -> str:
+ """Return a schema-qualified name for an arbitrary table."""
+ return f"{self.schema}.{table_name}"
+
+ @staticmethod
+ async def _hologres_reset(conn):
+ """Custom reset for Hologres connections."""
+ await conn.execute(
+ """
+ SELECT pg_advisory_unlock_all();
+ CLOSE ALL;
+ RESET ALL;
+ """,
+ )
+
+ async def _get_pool(self) -> Pool:
+ """Create or return the existing asyncpg connection pool."""
+ if self._pool is None:
+ if self.dsn:
+ self._pool = await asyncpg.create_pool(
+ dsn=self.dsn,
+ min_size=self.min_size,
+ max_size=self.max_size,
+ reset=self._hologres_reset,
+ )
+ else:
+ self._pool = await asyncpg.create_pool(
+ host=self.host,
+ port=self.port,
+ database=self.database,
+ user=self.user,
+ password=self.password,
+ min_size=self.min_size,
+ max_size=self.max_size,
+ reset=self._hologres_reset,
+ )
+
+ # Ensure schema exists
+ async with self._pool.acquire() as conn:
+ await conn.execute(f"CREATE SCHEMA IF NOT EXISTS {self.schema}")
+
+ logger.info(f"Hologres connection pool created for database {self.database}")
+
+ return self._pool
+
+ @staticmethod
+ def _vector_to_pg_array(vector: list[float]) -> str:
+ """Convert a Python list of floats to PostgreSQL array literal format."""
+ return "{" + ",".join(map(str, vector)) + "}"
+
+ @staticmethod
+ def _pg_array_to_vector(pg_array) -> list[float] | None:
+ """Convert a PostgreSQL array result to a Python list of floats."""
+ if pg_array is None:
+ return None
+ if isinstance(pg_array, list):
+ return [float(x) for x in pg_array]
+ # Handle string format like {1.0,2.0,3.0}
+ raw = str(pg_array)
+ if raw.startswith("{") and raw.endswith("}"):
+ return [float(x) for x in raw[1:-1].split(",")]
+ return None
+
+ async def list_collections(self) -> list[str]:
+ """List all available table names in the current schema."""
+ pool = await self._get_pool()
+ async with pool.acquire() as conn:
+ rows = await conn.fetch(
+ "SELECT table_name FROM information_schema.tables WHERE table_schema = $1",
+ self.schema,
+ )
+ return [row["table_name"] for row in rows]
+
+ async def create_collection(self, collection_name: str, **kwargs):
+ """Create a new Hologres table with vector support and HGraph index."""
+ self._validate_table_name(collection_name)
+ pool = await self._get_pool()
+ dimensions = kwargs.get("dimensions", self.embedding_model_dims)
+ qualified = self._qualify(collection_name)
+
+ async with pool.acquire() as conn:
+ create_sql = f"""
+ CREATE TABLE IF NOT EXISTS {qualified} (
+ id TEXT PRIMARY KEY,
+ content TEXT,
+ vector float4[] CHECK (array_ndims(vector) = 1 AND array_length(vector, 1) = {dimensions}),
+ metadata JSONB
+ )
+ WITH (
+ vectors = '{{
+ "vector": {{
+ "algorithm": "HGraph",
+ "distance_method": "{self.distance_method}",
+ "builder_params": {{
+ "base_quantization_type": "rabitq",
+ "rabitq_use_fht":true,
+ "graph_storage_type": "compressed",
+ "max_total_size_to_merge_mb": 4096,
+ "max_degree": 64,
+ "ef_construction": 400,
+ "precise_quantization_type": "fp32",
+ "use_reorder": true
+ }}
+ }}
+ }}'
+ )
+ """
+ await conn.execute(create_sql)
+
+ logger.info(f"Created Hologres collection {qualified} with dimensions={dimensions}")
+
+ async def delete_collection(self, collection_name: str, **kwargs):
+ """Remove the specified collection table from the database."""
+ self._validate_table_name(collection_name)
+ pool = await self._get_pool()
+ qualified = self._qualify(collection_name)
+ async with pool.acquire() as conn:
+ await conn.execute(f"DROP TABLE IF EXISTS {qualified}")
+ logger.info(f"Deleted collection {qualified}")
+
+ async def copy_collection(self, collection_name: str, **kwargs):
+ """Duplicate the structure and content of the current collection to a new table."""
+ self._validate_table_name(collection_name)
+ pool = await self._get_pool()
+ qualified_src = self._qualified_name
+ qualified_dst = self._qualify(collection_name)
+
+ async with pool.acquire() as conn:
+ columns = await conn.fetch(
+ """
+ SELECT column_name, data_type, udt_name
+ FROM information_schema.columns
+ WHERE table_name = $1 AND table_schema = $2
+ """,
+ self.collection_name,
+ self.schema,
+ )
+
+ if not columns:
+ raise ValueError(f"Source collection {qualified_src} does not exist")
+
+ # Create new table with primary key, then add data
+ await conn.execute(
+ f"""
+ SET hg_experimental_enable_create_table_like_properties = true;
+ CALL hg_create_table_like('{qualified_dst}', 'select * from {qualified_src}')
+ """,
+ )
+ await conn.execute(f"INSERT INTO {qualified_dst} SELECT * FROM {qualified_src} ;")
+
+ logger.info(f"Copied collection {qualified_src} to {qualified_dst}")
+
+ async def insert(self, nodes: VectorNode | list[VectorNode], **kwargs):
+ """Insert or upsert vector nodes into the Hologres collection."""
+ if isinstance(nodes, VectorNode):
+ nodes = [nodes]
+
+ if not nodes:
+ return
+
+ nodes_without_vectors = [node for node in nodes if node.vector is None]
+ if nodes_without_vectors:
+ nodes_with_vectors = await self.get_node_embeddings(nodes_without_vectors)
+ vector_map = {n.vector_id: n for n in nodes_with_vectors}
+ nodes_to_insert = [vector_map.get(n.vector_id, n) if n.vector is None else n for n in nodes]
+ else:
+ nodes_to_insert = nodes
+
+ pool = await self._get_pool()
+ data = [
+ (
+ node.vector_id,
+ node.content,
+ node.vector,
+ json.dumps(node.metadata),
+ )
+ for node in nodes_to_insert
+ ]
+
+ async with pool.acquire() as conn:
+ on_conflict = kwargs.get("on_conflict", "update")
+
+ if on_conflict == "update":
+ await conn.executemany(
+ f"""
+ INSERT INTO {self._qualified_name} (id, content, vector, metadata)
+ VALUES ($1, $2, $3::float4[], $4::jsonb)
+ ON CONFLICT (id) DO UPDATE SET
+ content = EXCLUDED.content,
+ vector = EXCLUDED.vector,
+ metadata = EXCLUDED.metadata
+ """,
+ data,
+ )
+ elif on_conflict == "ignore":
+ await conn.executemany(
+ f"""
+ INSERT INTO {self._qualified_name} (id, content, vector, metadata)
+ VALUES ($1, $2, $3::float4[], $4::jsonb)
+ ON CONFLICT (id) DO NOTHING
+ """,
+ data,
+ )
+ else:
+ await conn.executemany(
+ f"""
+ INSERT INTO {self._qualified_name} (id, content, vector, metadata)
+ VALUES ($1, $2, $3::float4[], $4::jsonb)
+ """,
+ data,
+ )
+
+ logger.info(f"Inserted {len(nodes_to_insert)} documents into {self._qualified_name}")
+
+ @staticmethod
+ def _build_filter_clause(filters: dict | None) -> tuple[str, list]:
+ """Generate an SQL WHERE clause and parameter list from a filter dictionary.
+
+ Supports two filter formats:
+ 1. Range query: {"field": [start_value, end_value]}
+ 2. Exact match: {"field": value}
+ """
+ if not filters:
+ return "", []
+
+ conditions = []
+ params = []
+ param_idx = 1
+
+ for key, value in filters.items():
+ if not key.replace("_", "").replace(".", "").isalnum():
+ raise ValueError(
+ f"Invalid metadata key: {key}. Only alphanumeric characters, underscore and dot are allowed.",
+ )
+
+ if isinstance(value, list) and len(value) == 2:
+ if isinstance(value[0], (int, float)) and isinstance(value[1], (int, float)):
+ conditions.append(
+ f"(metadata->>'{key}')::numeric >= ${param_idx} AND "
+ f"(metadata->>'{key}')::numeric <= ${param_idx + 1}",
+ )
+ else:
+ conditions.append(f"metadata->>'{key}' >= ${param_idx} AND metadata->>'{key}' <= ${param_idx + 1}")
+ params.extend([value[0], value[1]])
+ param_idx += 2
+ else:
+ conditions.append(f"metadata->>'{key}' = ${param_idx}")
+ params.append(str(value))
+ param_idx += 1
+
+ filter_clause = "WHERE " + " AND ".join(conditions) if conditions else ""
+ return filter_clause, params
+
+ async def search(
+ self,
+ query: str,
+ limit: int = 5,
+ filters: dict | None = None,
+ **kwargs,
+ ) -> list[VectorNode]:
+ """Perform vector similarity search using Hologres approx_cosine_distance."""
+ query_vector = await self.get_embedding(query)
+ vector_str = self._vector_to_pg_array(query_vector)
+ pool = await self._get_pool()
+
+ filter_clause, filter_params = self._build_filter_clause(filters)
+
+ # filter_params use $1..$N, limit uses $(N+1)
+ limit_placeholder = f"${len(filter_params) + 1}"
+
+ async with pool.acquire() as conn:
+ sql = f"""
+ SELECT id, content, vector, metadata,
+ approx_cosine_distance(vector, '{vector_str}') AS distance
+ FROM {self._qualified_name}
+ {filter_clause}
+ ORDER BY distance DESC
+ LIMIT {limit_placeholder}
+ """
+ rows = await conn.fetch(sql, *filter_params, limit)
+
+ results = []
+ score_threshold = kwargs.get("score_threshold")
+
+ for row in rows:
+ distance = float(row["distance"])
+ # approx_cosine_distance returns cosine similarity (higher = more similar)
+ score = distance
+ if score_threshold is not None and score < score_threshold:
+ continue
+
+ vector_data = self._pg_array_to_vector(row["vector"])
+
+ metadata = row["metadata"] if row["metadata"] else {}
+ if isinstance(metadata, str):
+ metadata = json.loads(metadata)
+
+ metadata["score"] = score
+ metadata["_distance"] = 1 - score
+
+ node = VectorNode(
+ vector_id=row["id"],
+ content=row["content"] or "",
+ vector=vector_data,
+ metadata=metadata,
+ )
+ results.append(node)
+
+ return results
+
+ async def delete(self, vector_ids: str | list[str], **kwargs):
+ """Remove specific vector records from the collection by their IDs."""
+ if isinstance(vector_ids, str):
+ vector_ids = [vector_ids]
+
+ if not vector_ids:
+ return
+
+ pool = await self._get_pool()
+ async with pool.acquire() as conn:
+ placeholders = ", ".join([f"${i + 1}" for i in range(len(vector_ids))])
+ await conn.execute(
+ f"DELETE FROM {self._qualified_name} WHERE id IN ({placeholders})",
+ *vector_ids,
+ )
+
+ logger.info(f"Deleted {len(vector_ids)} documents from {self._qualified_name}")
+
+ async def delete_all(self, **kwargs):
+ """Remove all vectors from the collection."""
+ pool = await self._get_pool()
+ async with pool.acquire() as conn:
+ result = await conn.execute(f"DELETE FROM {self._qualified_name}")
+
+ logger.info(f"Deleted all documents from {self._qualified_name} result={result}")
+
+ async def update(self, nodes: VectorNode | list[VectorNode], **kwargs):
+ """Update existing vector nodes with new content, embeddings, or metadata."""
+ if isinstance(nodes, VectorNode):
+ nodes = [nodes]
+
+ if not nodes:
+ return
+
+ nodes_without_vectors = [node for node in nodes if node.vector is None and node.content]
+ if nodes_without_vectors:
+ nodes_with_vectors = await self.get_node_embeddings(nodes_without_vectors)
+ vector_map = {n.vector_id: n for n in nodes_with_vectors}
+ nodes_to_update = [vector_map.get(n.vector_id, n) if n.vector is None and n.content else n for n in nodes]
+ else:
+ nodes_to_update = nodes
+
+ pool = await self._get_pool()
+ async with pool.acquire() as conn:
+ for node in nodes_to_update:
+ update_fields = []
+ params = []
+ idx = 1
+
+ if node.content:
+ update_fields.append(f"content = ${idx}")
+ params.append(node.content)
+ idx += 1
+
+ if node.vector:
+ update_fields.append(f"vector = ${idx}::float4[]")
+ params.append(node.vector)
+ idx += 1
+
+ if node.metadata:
+ update_fields.append(f"metadata = ${idx}::jsonb")
+ params.append(json.dumps(node.metadata))
+ idx += 1
+
+ if update_fields:
+ params.append(node.vector_id)
+ await conn.execute(
+ f"UPDATE {self._qualified_name} SET {', '.join(update_fields)} WHERE id = ${idx}",
+ *params,
+ )
+
+ logger.info(f"Updated {len(nodes_to_update)} documents in {self._qualified_name}")
+
+ async def get(self, vector_ids: str | list[str]) -> VectorNode | list[VectorNode] | None:
+ """Retrieve vector nodes by their unique identifiers."""
+ single_result = isinstance(vector_ids, str)
+ if single_result:
+ vector_ids = [vector_ids]
+
+ if not vector_ids:
+ return [] if not single_result else None
+
+ pool = await self._get_pool()
+ async with pool.acquire() as conn:
+ placeholders = ", ".join([f"${i + 1}" for i in range(len(vector_ids))])
+ rows = await conn.fetch(
+ f"SELECT id, content, vector, metadata FROM {self._qualified_name} WHERE id IN ({placeholders})",
+ *vector_ids,
+ )
+
+ results = []
+ for row in rows:
+ vector_data = self._pg_array_to_vector(row["vector"])
+
+ metadata = row["metadata"] if row["metadata"] else {}
+ if isinstance(metadata, str):
+ metadata = json.loads(metadata)
+
+ results.append(
+ VectorNode(
+ vector_id=row["id"],
+ content=row["content"] or "",
+ vector=vector_data,
+ metadata=metadata,
+ ),
+ )
+
+ if single_result:
+ return results[0] if results else None
+ return results
+
+ async def list(
+ self,
+ filters: dict | None = None,
+ limit: int | None = None,
+ sort_key: str | None = None,
+ reverse: bool = False,
+ ) -> list[VectorNode]:
+ """Return a list of vector nodes matching the provided filters and limit.
+
+ Args:
+ filters: Dictionary of filter conditions to match vectors
+ limit: Maximum number of vectors to return
+ sort_key: Key to sort the results by (e.g., field name in metadata). None for no sorting
+ reverse: If True, sort in descending order; if False, sort in ascending order
+ """
+ pool = await self._get_pool()
+ filter_clause, filter_params = self._build_filter_clause(filters)
+
+ order_clause = ""
+ if sort_key:
+ order_direction = "DESC" if reverse else "ASC"
+ order_clause = f"ORDER BY metadata->>'{sort_key}' {order_direction}"
+
+ limit_clause = ""
+ if limit:
+ limit_clause = f"LIMIT ${len(filter_params) + 1}"
+ filter_params.append(limit)
+
+ async with pool.acquire() as conn:
+ sql = f"""
+ SELECT id, content, vector, metadata
+ FROM {self._qualified_name}
+ {filter_clause}
+ {order_clause}
+ {limit_clause}
+ """
+ rows = await conn.fetch(sql, *filter_params)
+
+ results = []
+ for row in rows:
+ vector_data = self._pg_array_to_vector(row["vector"])
+
+ metadata = row["metadata"] if row["metadata"] else {}
+ if isinstance(metadata, str):
+ metadata = json.loads(metadata)
+
+ results.append(
+ VectorNode(
+ vector_id=row["id"],
+ content=row["content"] or "",
+ vector=vector_data,
+ metadata=metadata,
+ ),
+ )
+
+ return results
+
+ async def collection_info(self) -> dict[str, Any]:
+ """Fetch metadata including record count and disk usage for the collection."""
+ pool = await self._get_pool()
+ qualified = self._qualified_name
+
+ async with pool.acquire() as conn:
+ count = await conn.fetchval(f"SELECT COUNT(*) FROM {qualified}")
+ size = await conn.fetchval(f"SELECT pg_size_pretty(pg_total_relation_size('{qualified}'))")
+
+ return {
+ "name": qualified,
+ "count": count,
+ "size": size,
+ }
+
+ async def reset(self):
+ """Purge all data by dropping and recreating the collection table."""
+ logger.warning(f"Resetting collection {self._qualified_name}...")
+ await self.delete_collection(self.collection_name)
+ await self.create_collection(self.collection_name)
+
+ async def reset_collection(self, collection_name: str):
+ """Reset collection with table name validation."""
+ self._validate_table_name(collection_name)
+ self.collection_name = collection_name
+ await self.create_collection(collection_name)
+ logger.info(f"Collection reset to {self._qualified_name}")
+
+ async def start(self) -> None:
+ """Initialize the PGVector store.
+
+ Creates the connection pool and ensures the collection table exists.
+ """
+ await self._get_pool()
+ await super().start()
+ logger.info(f"Hologres collection {self._qualified_name} initialized")
+
+ async def close(self):
+ """Terminate the database connection pool."""
+ if self._pool is not None:
+ await self._pool.close()
+ self._pool = None
+ logger.info("Hologres connection pool closed")
diff --git a/tests/test_vector_store.py b/tests/test_vector_store.py
index 9b39ab9b..346be967 100644
--- a/tests/test_vector_store.py
+++ b/tests/test_vector_store.py
@@ -2,8 +2,8 @@
"""Unified test suite for vector store implementations.
This module provides comprehensive test coverage for LocalVectorStore, ESVectorStore,
-PGVectorStore, QdrantVectorStore, ChromaVectorStore, ObVecVectorStore and ZvecVectorStore implementations.
-Tests can be run for specific vector stores or all implementations.
+PGVectorStore, QdrantVectorStore, ChromaVectorStore, ObVecVectorStore, HologresVectorStore, and
+ZvecVectorStore implementations. Tests can be run for specific vector stores or all implementations.
Usage:
python test_vector_store.py --local # Test LocalVectorStore only
@@ -12,6 +12,7 @@ Usage:
python test_vector_store.py --qdrant # Test QdrantVectorStore only
python test_vector_store.py --chroma # Test ChromaVectorStore only
python test_vector_store.py --obvec # Test ObVecVectorStore only (needs seekdb / OceanBase)
+ python test_vector_store.py --hologres # Test HologresVectorStore only
python test_vector_store.py --zvec # Test ZvecVectorStore only
python test_vector_store.py --all # Test all vector stores
"""
@@ -32,6 +33,7 @@ from reme.core.utils import load_env, cosine_similarity
from reme.core.vector_store import (
BaseVectorStore,
ChromaVectorStore,
+ HologresVectorStore,
LocalVectorStore,
ESVectorStore,
ObVecVectorStore,
@@ -92,6 +94,19 @@ class TestConfig:
OBVEC_PASSWORD = os.environ.get("OBVEC_PASSWORD", "root")
OBVEC_DATABASE = os.environ.get("OBVEC_DATABASE", "test")
+ # HologresVectorStore settings
+ HOLOGRES_DSN = os.environ.get(
+ "HOLOGRES_DSN",
+ "",
+ ) # Full DSN connection string (overrides host/port/database/user/password)
+ HOLOGRES_HOST = os.environ.get("HOLOGRES_HOST", "localhost")
+ HOLOGRES_PORT = int(os.environ.get("HOLOGRES_PORT", "80"))
+ HOLOGRES_DATABASE = os.environ.get("HOLOGRES_DATABASE", "postgres")
+ HOLOGRES_USER = os.environ.get("HOLOGRES_USER", "postgres")
+ HOLOGRES_PASSWORD = os.environ.get("HOLOGRES_PASSWORD", "")
+ HOLOGRES_SCHEMA = os.environ.get("HOLOGRES_SCHEMA", "public")
+ HOLOGRES_MIN_SIZE = 1
+ HOLOGRES_MAX_SIZE = 5
# ZvecVectorStore settings
ZVEC_PATH = "./test_vector_store_zvec" # For local persistent mode
@@ -205,7 +220,7 @@ def get_store_type(store: BaseVectorStore) -> str:
store: Vector store instance
Returns:
- str: Type identifier ("local", "es", "pgvector", "qdrant", "chroma", "obvec", or "zvec")
+ str: Type identifier ("local", "es", "pgvector", "qdrant", "chroma", "obvec", "zvec", or "hologres")
"""
if isinstance(store, LocalVectorStore):
return "local"
@@ -221,6 +236,8 @@ def get_store_type(store: BaseVectorStore) -> str:
return "obvec"
elif isinstance(store, ZvecVectorStore):
return "zvec"
+ elif isinstance(store, HologresVectorStore):
+ return "hologres"
else:
raise ValueError(f"Unknown vector store type: {type(store)}")
@@ -230,7 +247,7 @@ def create_vector_store(store_type: str, collection_name: str) -> BaseVectorStor
"""Create a vector store instance based on type.
Args:
- store_type: Type of vector store ("local", "es", "pgvector", "qdrant", "chroma", or "obvec")
+ store_type: Type of vector store ("local", "es", "pgvector", "qdrant", "chroma", "obvec", or "hologres")
collection_name: Name of the collection
Returns:
@@ -312,6 +329,23 @@ def create_vector_store(store_type: str, collection_name: str) -> BaseVectorStor
dimension=config.EMBEDDING_DIMENSIONS,
distance="cosine",
)
+ elif store_type == "hologres":
+ kwargs = {
+ "collection_name": collection_name,
+ "embedding_model": embedding_model,
+ "db_path": tempfile.mkdtemp(prefix="test_hologres_"),
+ "host": config.HOLOGRES_HOST,
+ "port": config.HOLOGRES_PORT,
+ "database": config.HOLOGRES_DATABASE,
+ "user": config.HOLOGRES_USER,
+ "password": config.HOLOGRES_PASSWORD,
+ "schema": config.HOLOGRES_SCHEMA,
+ "min_size": config.HOLOGRES_MIN_SIZE,
+ "max_size": config.HOLOGRES_MAX_SIZE,
+ }
+ if config.HOLOGRES_DSN:
+ kwargs["dsn"] = config.HOLOGRES_DSN
+ return HologresVectorStore(**kwargs)
else:
raise ValueError(f"Unknown store type: {store_type}")
@@ -631,7 +665,7 @@ async def test_copy_collection(store: BaseVectorStore, store_name: str):
# Elasticsearch, PostgreSQL and OceanBase require lowercase table/index names
store_type = get_store_type(store)
- if store_type in ("es", "pgvector", "obvec"):
+ if store_type in ("es", "pgvector", "obvec", "hologres"):
copy_collection_name = copy_collection_name.lower()
# Clean up if exists
@@ -1835,6 +1869,7 @@ Examples:
python test_vector_store.py --qdrant # Test QdrantVectorStore only
python test_vector_store.py --chroma # Test ChromaVectorStore only
python test_vector_store.py --obvec # Test ObVecVectorStore (seekdb / OceanBase)
+ python test_vector_store.py --hologres # Test HologresVectorStore
python test_vector_store.py --all # Test all vector stores
""",
)
@@ -1868,6 +1903,11 @@ Examples:
action="store_true",
help="Test ObVecVectorStore",
)
+ parser.add_argument(
+ "--hologres",
+ action="store_true",
+ help="Test HologresVectorStore",
+ )
parser.add_argument(
"--zvec",
action="store_true",
@@ -1892,6 +1932,7 @@ Examples:
("qdrant", "QdrantVectorStore"),
("chroma", "ChromaVectorStore"),
("obvec", "ObVecVectorStore"),
+ ("hologres", "HologresVectorStore"),
("zvec", "ZvecVectorStore"),
]
else:
@@ -1908,6 +1949,8 @@ Examples:
stores_to_test.append(("chroma", "ChromaVectorStore"))
if args.obvec:
stores_to_test.append(("obvec", "ObVecVectorStore"))
+ if args.hologres:
+ stores_to_test.append(("hologres", "HologresVectorStore"))
if args.zvec:
stores_to_test.append(("zvec", "ZvecVectorStore"))
@@ -1920,11 +1963,12 @@ Examples:
("qdrant", "QdrantVectorStore"),
("chroma", "ChromaVectorStore"),
("obvec", "ObVecVectorStore"),
+ ("hologres", "HologresVectorStore"),
("zvec", "ZvecVectorStore"),
]
print("No vector store specified, defaulting to test all vector stores")
print(
- "Use --local/--es/--pgvector/--qdrant/--chroma/--obvec/--zvec to test specific ones\n",
+ "Use --local/--es/--pgvector/--qdrant/--chroma/--obvec/--zvec/--hologres to test specific ones\n",
)
# Run tests for each vector store
From ccadf1d3f9dd06ac05438f98acb5eb5f6dbcdb77 Mon Sep 17 00:00:00 2001
From: Aqil Aziz
Date: Thu, 14 May 2026 13:37:17 +0700
Subject: [PATCH 07/16] fix(file_watcher): reset stop event on restart (#233)
---
reme/core/file_watcher/base_file_watcher.py | 6 ++-
tests/test_base_file_watcher.py | 58 ++++++++++++---------
2 files changed, 37 insertions(+), 27 deletions(-)
diff --git a/reme/core/file_watcher/base_file_watcher.py b/reme/core/file_watcher/base_file_watcher.py
index b4644dd7..4269117c 100644
--- a/reme/core/file_watcher/base_file_watcher.py
+++ b/reme/core/file_watcher/base_file_watcher.py
@@ -76,12 +76,14 @@ class BaseFileWatcher:
if self._running:
return
+ self._stop_event = asyncio.Event()
self._running = True
async def _initialize_and_watch():
if self.rebuild_index_on_start:
- await self.file_store.clear_all()
- logger.info("Cleared all indexed data on start")
+ if self.file_store is not None:
+ await self.file_store.clear_all()
+ logger.info("Cleared all indexed data on start")
await self._scan_existing_files()
await self._watch_loop()
diff --git a/tests/test_base_file_watcher.py b/tests/test_base_file_watcher.py
index 234d03c2..dcf56abf 100644
--- a/tests/test_base_file_watcher.py
+++ b/tests/test_base_file_watcher.py
@@ -79,6 +79,15 @@ def temp_nested_dir(temp_dir: Path):
yield temp_dir
+def make_mock_file_store():
+ """Create an async-compatible mock file store."""
+ mock_file_store = MagicMock()
+ mock_file_store.clear_all = AsyncMock()
+ mock_file_store.list_files = AsyncMock(return_value=[])
+ mock_file_store.get_file_chunks = AsyncMock(return_value=[])
+ return mock_file_store
+
+
# ==================== Test Existing Paths ====================
@@ -147,6 +156,21 @@ class TestExistingPaths:
await watcher.close()
+ @pytest.mark.asyncio
+ async def test_restart_resets_stop_event(self, temp_dir: Path):
+ """Test restarting watcher resets the previous stop signal."""
+ watcher = BaseFileWatcher(watch_paths=str(temp_dir))
+
+ await watcher.start()
+ await watcher.close()
+
+ assert watcher._stop_event.is_set() is True
+
+ await watcher.start()
+ assert watcher._stop_event.is_set() is False
+
+ await watcher.close()
+
@pytest.mark.asyncio
async def test_multiple_start_calls(self, temp_dir: Path):
"""Test that multiple start calls don't create multiple tasks."""
@@ -381,9 +405,7 @@ class TestRebuildIndexOnStart:
callback_called.append(changes)
# Create mock file_store
- mock_file_store = MagicMock()
- mock_file_store.list_files = AsyncMock(return_value=[])
- mock_file_store.get_file_chunks = AsyncMock(return_value=[])
+ mock_file_store = make_mock_file_store()
watcher = BaseFileWatcher(
watch_paths=str(temp_dir),
@@ -408,9 +430,7 @@ class TestRebuildIndexOnStart:
callback_called.append(changes)
# Create mock file_store
- mock_file_store = MagicMock()
- mock_file_store.list_files = AsyncMock(return_value=[])
- mock_file_store.get_file_chunks = AsyncMock(return_value=[])
+ mock_file_store = make_mock_file_store()
watcher = BaseFileWatcher(
watch_paths=str(temp_dir),
@@ -441,9 +461,7 @@ class TestRebuildIndexOnStart:
async def callback(changes):
callback_called.append(changes)
- mock_file_store = MagicMock()
- mock_file_store.list_files = AsyncMock(return_value=[])
- mock_file_store.get_file_chunks = AsyncMock(return_value=[])
+ mock_file_store = make_mock_file_store()
watcher = BaseFileWatcher(
watch_paths=str(temp_dir),
@@ -474,9 +492,7 @@ class TestRebuildIndexOnStart:
async def callback(changes):
callback_called.append(changes)
- mock_file_store = MagicMock()
- mock_file_store.list_files = AsyncMock(return_value=[])
- mock_file_store.get_file_chunks = AsyncMock(return_value=[])
+ mock_file_store = make_mock_file_store()
watcher = BaseFileWatcher(
watch_paths=str(temp_nested_dir),
@@ -510,9 +526,7 @@ class TestRebuildIndexOnStart:
async def callback(changes):
callback_called.append(changes)
- mock_file_store = MagicMock()
- mock_file_store.list_files = AsyncMock(return_value=[])
- mock_file_store.get_file_chunks = AsyncMock(return_value=[])
+ mock_file_store = make_mock_file_store()
watcher = BaseFileWatcher(
watch_paths=str(temp_nested_dir),
@@ -545,9 +559,7 @@ class TestRebuildIndexOnStart:
async def callback(changes):
callback_called.append(changes)
- mock_file_store = MagicMock()
- mock_file_store.list_files = AsyncMock(return_value=[])
- mock_file_store.get_file_chunks = AsyncMock(return_value=[])
+ mock_file_store = make_mock_file_store()
watcher = BaseFileWatcher(
watch_paths="/nonexistent/path",
@@ -698,9 +710,7 @@ class TestEdgeCases:
"""Test watching a single file instead of directory."""
file_path = temp_files["txt_0"]
- mock_file_store = MagicMock()
- mock_file_store.list_files = AsyncMock(return_value=[])
- mock_file_store.get_file_chunks = AsyncMock(return_value=[])
+ mock_file_store = make_mock_file_store()
callback_called = []
@@ -731,9 +741,7 @@ class TestEdgeCases:
empty_dir = temp_dir / "empty"
empty_dir.mkdir()
- mock_file_store = MagicMock()
- mock_file_store.list_files = AsyncMock(return_value=[])
- mock_file_store.get_file_chunks = AsyncMock(return_value=[])
+ mock_file_store = make_mock_file_store()
callback_called = []
@@ -774,7 +782,7 @@ class TestEdgeCases:
unicode_dir.mkdir()
file_path = unicode_dir / "文件.txt"
- file_path.write_text("内容")
+ file_path.write_text("内容", encoding="utf-8")
watcher = BaseFileWatcher(
watch_paths=str(unicode_dir),
From 20b37414cbb0e0dde9a38d08f2aa59edc5af0515 Mon Sep 17 00:00:00 2001
From: "jinli.yl"
Date: Thu, 14 May 2026 14:40:07 +0800
Subject: [PATCH 08/16] fix(file-watcher): enable force polling for file
watcher
- Set force_polling=True in awatch to improve file detection reliability
- Bump version from 0.3.1.8 to 0.3.1.9
---
reme/__init__.py | 2 +-
reme/core/file_watcher/base_file_watcher.py | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/reme/__init__.py b/reme/__init__.py
index c1dbeb42..01261acd 100644
--- a/reme/__init__.py
+++ b/reme/__init__.py
@@ -6,7 +6,7 @@ from . import extension
from . import memory
from .reme import ReMe
-__version__ = "0.3.1.8"
+__version__ = "0.3.1.9"
__all__ = [
"config",
diff --git a/reme/core/file_watcher/base_file_watcher.py b/reme/core/file_watcher/base_file_watcher.py
index 4269117c..09b2f703 100644
--- a/reme/core/file_watcher/base_file_watcher.py
+++ b/reme/core/file_watcher/base_file_watcher.py
@@ -185,6 +185,7 @@ class BaseFileWatcher:
logger.info(f"Starting watch on valid paths: {valid_paths}")
async for changes in awatch(
*valid_paths,
+ force_polling=True,
watch_filter=self.watch_filter,
recursive=self.recursive,
debounce=self.debounce,
From e411eeb4c0472df3b4b525e972d7d7748e336578 Mon Sep 17 00:00:00 2001
From: jinliyl <6469360+jinliyl@users.noreply.github.com>
Date: Sun, 17 May 2026 14:14:43 +0800
Subject: [PATCH 09/16] dev/reme4 init merge (#236)
---
.github/workflows/unittest.yml | 43 +
docs4/reme_design.md | 131 ++
docs4/todo.md | 7 +
pyproject.toml | 26 +-
reme4/__init__.py | 26 +
reme4/application.py | 174 ++
reme4/components/__init__.py | 43 +
reme4/components/application_context.py | 28 +
reme4/components/as_llm/__init__.py | 53 +
reme4/components/as_llm_formatter/__init__.py | 44 +
.../reme_openai_chat_formatter.py | 141 ++
reme4/components/as_token_counter/__init__.py | 35 +
.../estimate_token_counter.py | 21 +
reme4/components/base_component.py | 185 +++
reme4/components/client/__init__.py | 7 +
reme4/components/client/base_client.py | 41 +
reme4/components/client/http_client.py | 143 ++
reme4/components/client/mcp_client.py | 125 ++
reme4/components/component_registry.py | 77 +
reme4/components/embedding/__init__.py | 6 +
.../embedding/base_embedding_model.py | 214 +++
.../embedding/openai_embedding_model.py | 52 +
reme4/components/file_graph/__init__.py | 7 +
.../components/file_graph/base_file_graph.py | 53 +
.../components/file_graph/local_file_graph.py | 138 ++
reme4/components/file_graph/nx_file_graph.py | 122 ++
reme4/components/file_parser/__init__.py | 7 +
.../file_parser/bare_file_parser.py | 22 +
.../file_parser/base_file_parser.py | 30 +
.../file_parser/default_file_parser.py | 123 ++
reme4/components/file_store/__init__.py | 14 +
.../components/file_store/base_file_store.py | 100 ++
.../components/file_store/local_file_store.py | 177 +++
reme4/components/file_watcher/__init__.py | 9 +
.../file_watcher/base_file_watcher.py | 118 ++
.../file_watcher/lite_file_watcher.py | 129 ++
reme4/components/job/__init__.py | 6 +
reme4/components/job/base_job.py | 53 +
reme4/components/job/stream_job.py | 21 +
reme4/components/keyword_index/__init__.py | 6 +
.../keyword_index/base_keyword_index.py | 70 +
reme4/components/keyword_index/bm25_index.py | 206 +++
reme4/components/prompt_handler.py | 125 ++
reme4/components/runtime_context.py | 87 +
reme4/components/service/__init__.py | 11 +
reme4/components/service/base_service.py | 48 +
reme4/components/service/http_service.py | 99 ++
reme4/components/service/mcp_service.py | 71 +
reme4/components/tokenizer/__init__.py | 11 +
reme4/components/tokenizer/base_tokenizer.py | 44 +
reme4/components/tokenizer/jieba_tokenizer.py | 27 +
reme4/components/tokenizer/regex_tokenizer.py | 31 +
reme4/components/tokenizer/stopwords | 1395 +++++++++++++++++
reme4/config/__init__.py | 8 +
reme4/config/config_parser.py | 219 +++
reme4/config/default.yaml | 169 ++
reme4/constants.py | 7 +
reme4/enumeration/__init__.py | 9 +
reme4/enumeration/chunk_enum.py | 21 +
reme4/enumeration/component_enum.py | 37 +
reme4/reme.py | 43 +
reme4/schema/__init__.py | 25 +
reme4/schema/application_config.py | 45 +
reme4/schema/emb_node.py | 34 +
reme4/schema/file_chunk.py | 26 +
reme4/schema/file_front_matter.py | 24 +
reme4/schema/file_link.py | 18 +
reme4/schema/file_node.py | 16 +
reme4/schema/request.py | 11 +
reme4/schema/response.py | 15 +
reme4/schema/stream_chunk.py | 14 +
reme4/steps/__init__.py | 9 +
reme4/steps/base_step.py | 140 ++
reme4/steps/common/__init__.py | 21 +
reme4/steps/common/demo.py | 53 +
reme4/steps/common/health_check.py | 164 ++
reme4/steps/common/help.py | 42 +
reme4/steps/common/reindex.py | 24 +
reme4/steps/common/search.py | 227 +++
reme4/steps/common/stream_demo.py | 42 +
reme4/steps/common/version.py | 19 +
reme4/utils/__init__.py | 31 +
reme4/utils/common_utils.py | 249 +++
reme4/utils/env_utils.py | 36 +
reme4/utils/logger_utils.py | 108 ++
reme4/utils/logo_utils.py | 87 +
reme4/utils/service_utils.py | 96 ++
reme4/utils/similarity_utils.py | 35 +
tests4/unittest/test_bm25_index_perf.py | 337 ++++
tests4/unittest/test_bm25_lite.py | 586 +++++++
tests4/unittest/test_common_steps.py | 290 ++++
tests4/unittest/test_default_file_parser.py | 387 +++++
tests4/unittest/test_file_graph.py | 333 ++++
tests4/unittest/test_file_store.py | 330 ++++
tests4/unittest/test_file_watcher.py | 349 +++++
tests4/unittest/test_tokenizer.py | 169 ++
96 files changed, 9880 insertions(+), 7 deletions(-)
create mode 100644 .github/workflows/unittest.yml
create mode 100644 docs4/reme_design.md
create mode 100644 docs4/todo.md
create mode 100644 reme4/__init__.py
create mode 100644 reme4/application.py
create mode 100644 reme4/components/__init__.py
create mode 100644 reme4/components/application_context.py
create mode 100644 reme4/components/as_llm/__init__.py
create mode 100644 reme4/components/as_llm_formatter/__init__.py
create mode 100644 reme4/components/as_llm_formatter/reme_openai_chat_formatter.py
create mode 100644 reme4/components/as_token_counter/__init__.py
create mode 100644 reme4/components/as_token_counter/estimate_token_counter.py
create mode 100644 reme4/components/base_component.py
create mode 100644 reme4/components/client/__init__.py
create mode 100644 reme4/components/client/base_client.py
create mode 100644 reme4/components/client/http_client.py
create mode 100644 reme4/components/client/mcp_client.py
create mode 100644 reme4/components/component_registry.py
create mode 100644 reme4/components/embedding/__init__.py
create mode 100644 reme4/components/embedding/base_embedding_model.py
create mode 100644 reme4/components/embedding/openai_embedding_model.py
create mode 100644 reme4/components/file_graph/__init__.py
create mode 100644 reme4/components/file_graph/base_file_graph.py
create mode 100644 reme4/components/file_graph/local_file_graph.py
create mode 100644 reme4/components/file_graph/nx_file_graph.py
create mode 100644 reme4/components/file_parser/__init__.py
create mode 100644 reme4/components/file_parser/bare_file_parser.py
create mode 100644 reme4/components/file_parser/base_file_parser.py
create mode 100644 reme4/components/file_parser/default_file_parser.py
create mode 100644 reme4/components/file_store/__init__.py
create mode 100644 reme4/components/file_store/base_file_store.py
create mode 100644 reme4/components/file_store/local_file_store.py
create mode 100644 reme4/components/file_watcher/__init__.py
create mode 100644 reme4/components/file_watcher/base_file_watcher.py
create mode 100644 reme4/components/file_watcher/lite_file_watcher.py
create mode 100644 reme4/components/job/__init__.py
create mode 100644 reme4/components/job/base_job.py
create mode 100644 reme4/components/job/stream_job.py
create mode 100644 reme4/components/keyword_index/__init__.py
create mode 100644 reme4/components/keyword_index/base_keyword_index.py
create mode 100644 reme4/components/keyword_index/bm25_index.py
create mode 100644 reme4/components/prompt_handler.py
create mode 100644 reme4/components/runtime_context.py
create mode 100644 reme4/components/service/__init__.py
create mode 100644 reme4/components/service/base_service.py
create mode 100644 reme4/components/service/http_service.py
create mode 100644 reme4/components/service/mcp_service.py
create mode 100644 reme4/components/tokenizer/__init__.py
create mode 100644 reme4/components/tokenizer/base_tokenizer.py
create mode 100644 reme4/components/tokenizer/jieba_tokenizer.py
create mode 100644 reme4/components/tokenizer/regex_tokenizer.py
create mode 100644 reme4/components/tokenizer/stopwords
create mode 100644 reme4/config/__init__.py
create mode 100644 reme4/config/config_parser.py
create mode 100644 reme4/config/default.yaml
create mode 100644 reme4/constants.py
create mode 100644 reme4/enumeration/__init__.py
create mode 100644 reme4/enumeration/chunk_enum.py
create mode 100644 reme4/enumeration/component_enum.py
create mode 100644 reme4/reme.py
create mode 100644 reme4/schema/__init__.py
create mode 100644 reme4/schema/application_config.py
create mode 100644 reme4/schema/emb_node.py
create mode 100644 reme4/schema/file_chunk.py
create mode 100644 reme4/schema/file_front_matter.py
create mode 100644 reme4/schema/file_link.py
create mode 100644 reme4/schema/file_node.py
create mode 100644 reme4/schema/request.py
create mode 100644 reme4/schema/response.py
create mode 100644 reme4/schema/stream_chunk.py
create mode 100644 reme4/steps/__init__.py
create mode 100644 reme4/steps/base_step.py
create mode 100644 reme4/steps/common/__init__.py
create mode 100644 reme4/steps/common/demo.py
create mode 100644 reme4/steps/common/health_check.py
create mode 100644 reme4/steps/common/help.py
create mode 100644 reme4/steps/common/reindex.py
create mode 100644 reme4/steps/common/search.py
create mode 100644 reme4/steps/common/stream_demo.py
create mode 100644 reme4/steps/common/version.py
create mode 100644 reme4/utils/__init__.py
create mode 100644 reme4/utils/common_utils.py
create mode 100644 reme4/utils/env_utils.py
create mode 100644 reme4/utils/logger_utils.py
create mode 100644 reme4/utils/logo_utils.py
create mode 100644 reme4/utils/service_utils.py
create mode 100644 reme4/utils/similarity_utils.py
create mode 100644 tests4/unittest/test_bm25_index_perf.py
create mode 100644 tests4/unittest/test_bm25_lite.py
create mode 100644 tests4/unittest/test_common_steps.py
create mode 100644 tests4/unittest/test_default_file_parser.py
create mode 100644 tests4/unittest/test_file_graph.py
create mode 100644 tests4/unittest/test_file_store.py
create mode 100644 tests4/unittest/test_file_watcher.py
create mode 100644 tests4/unittest/test_tokenizer.py
diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
new file mode 100644
index 00000000..263bd23a
--- /dev/null
+++ b/.github/workflows/unittest.yml
@@ -0,0 +1,43 @@
+name: Tests ReMe
+
+on:
+ push:
+ branches: [main, master, dev, develop]
+ pull_request:
+ branches: [main, master, dev, develop]
+ workflow_dispatch:
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ unit-tests:
+ name: Unit Tests - py${{ matrix.python-version }}
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: ["3.10", "3.13"]
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
+ cache: 'pip'
+
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip setuptools wheel
+ pip install -e ".[dev,core]"
+
+ - name: Run tests4 unit tests
+ run: |
+ pytest tests4/unittest \
+ -v \
+ --tb=long \
+ -s \
+ --log-cli-level=WARNING
diff --git a/docs4/reme_design.md b/docs4/reme_design.md
new file mode 100644
index 00000000..66acb982
--- /dev/null
+++ b/docs4/reme_design.md
@@ -0,0 +1,131 @@
+# 快速测试
+
+```bash
+# 终端 A：启动服务
+reme4 start
+
+# 终端 B：调用 version 验证服务可用
+reme4 version
+# 预期输出：✅ ReMe v{__version__}
+```
+
+# 基础Job
+
+@jinli
+说明：📥 输入参数 ｜ 📤 输出 ｜ ⭐ 必填 ｜ 🎚️ 默认值 ｜ 🛠️ 内部行为
+
+| 分类 | 能力 (register name) | 参数 & 行为 |
+|-----------|--------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 🌐 通用 | 🆘 `help` (`help_step`) | 📥 无 ｜ 📤 `answer` 一行一个 job：`🛠️ \`{name}\` — {description} 📥 {params}`，参数渲染为 `name:type*`(必填) / `name:type={default}` / `name:type` ｜ 📊 `metadata.job_count` ｜ 🛠️ 自动跳过名为 `help` 的 job |
+| 🌐 通用 | 🩺 `health_check` (`health_check_step`) | 📥 无 ｜ 📤 `answer = "✅/❌ ReMe v{version} - healthy/unhealthy"` ｜ 📊 `metadata.health = {version, healthy, components}` ｜ 🧩 覆盖组件：`embedding_model`(🟢 is_started/is_healthy/model_name/dimensions/cache_size/memory) · `file_graph`(🕸️ n_nodes/n_edges/n_virtual\|n_pending/memory) · `file_store`(📦 n_chunks/n_chunks_with_embedding/memory) · `file_watcher`(👀 background_running/watch_paths) · `keyword_index`(🔤 n_docs/vocab_size/memory) ｜ 🛠️ deep sizeof（含 numpy.nbytes），未启动 / 后台未跑 / embedding 不健康 → ❌ |
+| 🌐 通用 | 🏷️ `version` (`version_step`) | 📥 无 ｜ 📤 `answer = reme4.__version__` ｜ 📊 `metadata.version` |
+| 🌐 通用 | 🔄 `reindex` (`reindex_step`) | 📥 无 ｜ 📤 `answer = "🔄 Reindexed {added} file(s)"` ｜ 📊 `metadata.counts = {added, ...}` ｜ 🛠️ 流程：`file_watcher.close()` → `file_store.clear()` → `file_watcher.update_store()` → `file_watcher.start()`（finally 保证重启） |
+| 🔎 search | 🔍 `search` (`search_step`) | 📥 `query:str` ⭐ ｜ 🎚️ `limit:int=5`(>0) ｜ 🎚️ `min_score:float=0.0` ｜ ⚖️ `vector_weight:float=0.7` ∈[0,1]（keyword 权 = 1-vw）｜ 🔀 `candidate_multiplier:float=3.0`（candidates = min(200, limit×mult)）｜ 🔗 `expand_links:bool=True` ｜ 🔢 `max_links_per_direction:int=10` ｜ 🎚️ `search_filter:dict={}` ｜ 📤 `answer` 每命中一行 `path:start-end [score=… vector=… keyword=…] text` + 缩进的 `→ outlinks (n)` / `← inlinks (n)` + `via predicate=… anchor=#…` ｜ 📊 `metadata.results` / `metadata.link_expansion` / `metadata.counts={vector,keyword,returned,hybrid}` ｜ 🛠️ 并行 `vector_search` + `keyword_search` → RRF 融合（K=60，按 chunk.id 合并）→ `min_score` 过滤 → `limit` 截断 → 邻居 meta 注入 |
+| 🧪 demo | 🪄 `demo_echo` (`demo_echo_step1` + `step2`) | 📥 `query:str=""` ｜ 🎚️ `min_score:float=0.5` ｜ 🛠️ step1：`processed_query = query.strip().lower()`，`adjusted_min_score = min_score * 0.9`，写回 context ｜ 📤 step2：`answer = "echo: {processed_query} (min_score={adjusted_min_score})"` ｜ 📊 `metadata = {step, query, min_score, processed_query, adjusted_min_score}` |
+| 🌊 demo | 🌊 `stream_demo` (`stream_demo_step1` + `step2`) | 📥 `query:str=""` ｜ 🎚️ `repeat:int=10` ｜ 🎚️ `interval:float=0.1`（秒/字符）｜ 🛠️ step1：`stream_text = query * repeat` 写回 context ｜ 📤 step2：按字符 `add_stream_string(ch, ChunkEnum.CONTENT)` 流式输出，`asyncio.sleep(interval)` 节流 |
+
+@sen
+| tags | stat | 返回特定tag信息 |
+| tags | list | 返回所有tag列表 |
+| crud | upload/download | 其他文件 |
+| file | stat | path |
+| file | list | path |
+| property | property:read | |
+| property | property:update | path="My Note" status=done xx=xxx |
+| property | property:delete | keys="[xxxx, xxxx]" |
+| graph | traverse | path="My Note" directtion=forward/backward depth=1 predicat=xxx |
+
+@wangce
+| crud | create | path="New Note" content="# Hello" title="xxx" tags="[]" status="" |
+| crud | read | path="Templates/Recipe.md" |
+| crud | edit | path="Templates/Recipe.md" old="xxx" new="xxx" |
+| crud | append | path="My Note" content="New line" |
+| crud | prepend | path="My Note" content="New line" |
+| crud | delete | path="My Note
+| daily:crud | daily:xxx | 与 crud 参数保持一致 |
+
+# 日记类型
+
+| 类型 | 路径 | 说明 |
+|-----------|-----------------------------------------------|-----------------------------|
+| daily | {daily}/xxxx-mm-dd.md + xxxx-mm-dd/{event}.md | 按日期归档的原始信息记录 |
+| topic | topic/{topic:-personal(agent)}/{xxxx}.md | 按主题聚类的二次加工内容 |
+| proactive | todo | 基于 daily / topic 思考后主动推送的消息 |
+
+# 生成Job
+
+| 任务 | 输入 | 输出 | 触发时机 | 说明 |
+|-------------------------|---------------|-----------------------------------------------|-----------------------------|------------------------------------------------------|
+| 日记summary @sen @wangce | msg | {daily}/xxxx-mm-dd.md + xxxx-mm-dd/{event}.md | freq (every_n_turn、compact) | 把 msg 的信息写入 daily 目录 |
+| 主题dream + 生成链接 @sen | daily/xxx | knowledge/xxx | /dream | 把 daily 目录的内容按主题聚类合并到 topic 目录, 主动在文档中建立 [[link]] 关联 |
+| 主动proactive @wangce | daily / topic | proactive_query | pre_query | 思考 daily / topic 信息，主动决定推送给用户的消息 |
+
+2. file_parser
+ a. 抽象基类 parse: @jinli
+ ⅰ. 输入是path：相对路径
+ ⅱ. 输出是FileMetadata & list[FileChunks] & list[FileEdge]
+ b. default parser 兼容老方案 @jinli
+ ⅰ. 带overlap的chunking策略 ，不输出FileEdge
+ c. markdown parser @sen
+ ⅰ. 根据markdown ast做chunk，不需要overlap
+ ⅱ. 增加一个索引的chunk chunk_type @锦鲤 file_chunk_type content/index
+ ⅲ. 增加link的正则解析：predicate:: [[path#anchor]]
+3. file_store @sen
+ a. 抽象存储：
+ ⅰ. filenode = file + path + st_mtime + metadata + list[FileEdge]
+ ⅱ. graph=dict[str, filenode] 内存+json
+ ⅲ. list[FileChunk] 存db
+ b. 抽象基类
+ ⅰ. graph：fellow dict的操作 update/get/set
+ ⅱ. chunks dict[str, list[chunk]]
+ 1. delete_chunks_by_path
+ 2. update_chunks_by_path
+ 3. list_chunks_by_path
+ 4. vector_search/keyword_search
+ ⅲ. 手写一个bm25检索
+ ⅳ. 【核心】检索机制 vector bm25 graph 如何进行融合
+4. file_watcher @jinli
+ a. 抽象基类
+ ⅰ. on_start:
+ 1. file_store 的start 在前，加载graph，file_watcher在后，递归扫描目录
+ a. 通过ms_time对比graph，on_change 进行改动
+ ⅱ. on_change:
+ 1. 更新/增加:
+ a. delete_chunks_by_path 更新数据库
+ b. upate_chunks_by_path 更新数据库
+ c. 更新graph
+ 2. 删除
+ a. delete_chunks_by_path 更新数据库
+
+MemorySchema
+
+1. markdown文件结构 @sen
+ a. formatter：
+ ⅰ. title
+ ⅱ. desc
+ ⅲ. tags
+ ⅳ.
+2. memory文件结构目录
+ a. MEMORY.md
+ b. msg/files -> daily/YYYYMMDD/YYYYMMDD.md + xxxx.md
+ ⅰ. YYYYMMDD.md
+ 1. xxx -> xxxx.md
+ 2. xxx -> xxxd.md
+ ⅱ.
+ c. daily -> topic/topic_l1/topic_l1.md + xxx.md + topic_l2
+ d. proactive
+
+steps:
+
+1. 治理（算法+LLM）：
+ a. 节点关联P0：现有的链接做补充，挖掘新的LLM的link
+ ⅰ. /Users/yuli/workspace/ReMe/reme2/component/edge_extractor/llm_edge_extractor.py
+ ⅱ. 移动到steps
+ b. 节点整合/节点拆分/节点归档
+ c. 健康度检查
+2. retrieve 调用store的检索
+3. 原子steps：reme edit
+4. 组合steps：总结：
+ a. - freq (every_n_turn、compact) -> daily_summarizer
+ b. topic (/dream ) -> topic_summarizer(daily_xx -> topic_xx)
+ c. proactive -> proactive_summarizer(personal_xxx -> proactive_query - pre_query
diff --git a/docs4/todo.md b/docs4/todo.md
new file mode 100644
index 00000000..938edcc1
--- /dev/null
+++ b/docs4/todo.md
@@ -0,0 +1,7 @@
+1. 完善mcp_servers config
+2. 完善mcp/http的服务测试
+3. [PosixPath('.reme')]
+4. error
+5. meta信息存在一个地方
+6. 测试一个完整的Service client的框架，测试各种命令
+7. config 默认改成default
diff --git a/pyproject.toml b/pyproject.toml
index 11177ffd..c9f644af 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -33,9 +33,7 @@ classifiers = [
keywords = ["llm", "memory", "experience", "memoryscope", "ai", "mcp", "http", "reme", "personal"]
dependencies = [
- "sqlite-vec>=0.1.6",
- "prompt_toolkit>=3.0.52",
- "rich>=14.2.0",
+ "aiofiles>=24.1.0",
"asyncpg>=0.31.0",
"chromadb>=1.3.5",
"dashscope>=1.25.1",
@@ -43,23 +41,28 @@ dependencies = [
"fastapi>=0.121.3",
"fastmcp>=2.14.1",
"httpx>=0.28.1",
+ "jieba>=0.42.1",
"loguru>=0.7.3",
"mcp>=1.25.0",
+ "networkx>=3.4",
"numpy>=2.2.6",
"openai>=2.8.1",
"pandas>=2.3.3",
+ "prompt_toolkit>=3.0.52",
"pydantic>=2.12.4",
"pyobvector>=0.1.20",
+ "pyyaml>=6.0.3",
+ "qdrant-client>=1.16.0",
+ "rich>=14.2.0",
+ "sqlite-vec>=0.1.6",
# pyobvector imports Expression from sqlglot; removed from sqlglot 30+ top-level API
"sqlglot>=25,<30",
- "qdrant-client>=1.16.0",
"tavily-python>=0.7.13",
"tiktoken>=0.12.0",
"tqdm>=4.67.1",
"transformers>=4.57.3",
"uvicorn>=0.40.0",
"watchfiles>=1.1.1",
- "pyyaml>=6.0.3",
]
[project.optional-dependencies]
@@ -75,6 +78,8 @@ dev = [
"furo",
"sphinxcontrib-mermaid",
"pre-commit",
+ "pytest>=8.0",
+ "pytest-asyncio>=0.23",
]
full = [
@@ -85,14 +90,14 @@ litellm = [
"litellm==1.80.0",
]
-light = [
+core = [
"agentscope==1.0.18",
"flowllm[reme]>=0.2.0.10",
]
[tool.setuptools.packages.find]
where = ["."]
-include = ["reme_ai*", "reme*"]
+include = ["reme_ai*", "reme*", "reme4*"]
exclude = ["test*", "cookbook*", "doc*", "library*", "dist*"]
[tool.setuptools.package-data]
@@ -108,6 +113,12 @@ reme = [
"**/*.json",
]
+reme4 = [
+ "**/*.yaml",
+ "**/*.py",
+ "**/*.json",
+]
+
[tool.setuptools.dynamic]
version = { attr = "reme.__version__" }
@@ -120,6 +131,7 @@ Repository = "https://github.com/agentscope-ai/ReMe"
reme = "reme_ai.main:main"
reme2 = "reme.reme:main"
remecli = "reme.reme_cli:main"
+reme4 = "reme4.reme:main"
[tool.pytest.ini_options]
asyncio_default_fixture_loop_scope = "function"
diff --git a/reme4/__init__.py b/reme4/__init__.py
new file mode 100644
index 00000000..e4db3de0
--- /dev/null
+++ b/reme4/__init__.py
@@ -0,0 +1,26 @@
+"""ReMe CLI package."""
+
+__version__ = "0.4.0.0"
+
+from . import config
+from . import constants
+from . import enumeration
+from . import schema
+from . import steps
+from . import utils
+from .application import Application
+from .components import BaseComponent
+from .reme import ReMe
+
+__all__ = [
+ "Application",
+ "BaseComponent",
+ "ReMe",
+ # submodules
+ "config",
+ "constants",
+ "enumeration",
+ "schema",
+ "steps",
+ "utils",
+]
diff --git a/reme4/application.py b/reme4/application.py
new file mode 100644
index 00000000..a06dadbd
--- /dev/null
+++ b/reme4/application.py
@@ -0,0 +1,174 @@
+"""Main application entry point."""
+
+import asyncio
+import heapq
+from pathlib import Path
+from typing import AsyncGenerator
+
+from .components import BaseComponent, ApplicationContext
+from .enumeration import ComponentEnum
+from .schema import Response, StreamChunk
+from .utils import execute_stream_task, print_logo, get_logger
+
+
+class Application(BaseComponent):
+ """Main application: initializes components, resolves dependencies, runs jobs."""
+
+ def __init__(self, **kwargs) -> None:
+ self.context = ApplicationContext(**kwargs)
+
+ working_path = Path(self.config.working_dir).absolute()
+ working_path.mkdir(parents=True, exist_ok=True)
+ (working_path / self.config.metadata_dir).mkdir(parents=True, exist_ok=True)
+ (working_path / self.config.daily_dir).mkdir(parents=True, exist_ok=True)
+ (working_path / self.config.knowledge_dir).mkdir(parents=True, exist_ok=True)
+
+ if self.config.enable_logo:
+ print_logo(self.config)
+
+ logger = get_logger(
+ log_to_console=self.config.log_to_console,
+ log_to_file=self.config.log_to_file,
+ force_init=True,
+ )
+ logger.info(f"Initializing {self.config.app_name} Application")
+ super().__init__()
+
+ from .components import R
+
+ # Service
+ service_config = self.config.service
+ if not service_config.backend:
+ raise ValueError("Service configuration is missing the required 'backend' field")
+ service_cls = R.get(ComponentEnum.SERVICE, service_config.backend)
+ if not service_cls:
+ raise ValueError(f"Unregistered service backend '{service_config.backend}'")
+ params = service_config.model_dump()
+ params["app_context"] = self.context
+ self.context.service = service_cls(**params)
+
+ # Components
+ for component_type, component_configs in self.config.components.items():
+ self.context.components[component_type] = {}
+ for name, config in component_configs.items():
+ if not config.backend:
+ raise ValueError(f"Component '{name}' is missing the required 'backend' field")
+ backend_cls = R.get(component_type, config.backend)
+ if not backend_cls:
+ raise ValueError(f"Unregistered backend '{config.backend}' for component '{name}'")
+ params = config.model_dump()
+ params.setdefault("name", name)
+ params["app_context"] = self.context
+ self.context.components[component_type][name] = backend_cls(**params)
+
+ # Jobs
+ for job_config in self.config.jobs:
+ if not job_config.backend:
+ raise ValueError(f"Job '{job_config.name}' is missing the required 'backend' field")
+ job_cls = R.get(ComponentEnum.JOB, job_config.backend)
+ if not job_cls:
+ raise ValueError(f"Unregistered backend '{job_config.backend}' for job '{job_config.name}'")
+ params = job_config.model_dump()
+ params["app_context"] = self.context
+ self.context.jobs[job_config.name] = job_cls(**params)
+
+ @property
+ def config(self):
+ """Application configuration."""
+ return self.context.app_config
+
+ def _topological_order(self) -> list[BaseComponent]:
+ """Kahn's algorithm. Raises on missing required dep or cycle."""
+ nodes: dict[tuple[ComponentEnum, str], BaseComponent] = {
+ (ctype, name): comp for ctype, group in self.context.components.items() for name, comp in group.items()
+ }
+
+ in_degree: dict[tuple[ComponentEnum, str], int] = dict.fromkeys(nodes, 0)
+ dependents: dict[tuple[ComponentEnum, str], list[tuple[ComponentEnum, str]]] = {k: [] for k in nodes}
+ for key, comp in nodes.items():
+ for dep in comp.dependencies:
+ dep_key = (dep.ctype, dep.name)
+ if dep_key in nodes:
+ dependents[dep_key].append(key)
+ in_degree[key] += 1
+ elif not dep.optional:
+ raise ValueError(
+ f"Component {key[0].value}:{key[1]} depends on {dep.ctype.value}:{dep.name}, not registered",
+ )
+
+ ready = [k for k, d in in_degree.items() if d == 0]
+ heapq.heapify(ready)
+ ordered: list[BaseComponent] = []
+ while ready:
+ key = heapq.heappop(ready)
+ ordered.append(nodes[key])
+ for downstream in dependents[key]:
+ in_degree[downstream] -= 1
+ if in_degree[downstream] == 0:
+ heapq.heappush(ready, downstream)
+
+ if len(ordered) != len(nodes):
+ unresolved = [f"{k[0].value}:{k[1]}" for k, d in in_degree.items() if d > 0]
+ raise ValueError(f"Circular dependency detected among: {unresolved}")
+ return ordered
+
+ async def _start(self) -> None:
+ """Start components in topological order, then jobs."""
+ start_order = self._topological_order()
+ order_str = " -> ".join(f"{c.component_type.value}:{c.name}" for c in start_order)
+ self.logger.info(f"Component start order: {order_str}")
+
+ for component in start_order:
+ try:
+ await component.start()
+ except Exception as e:
+ self.logger.exception(f"Failed to start {component.component_type.value}:{component.name}: {e}")
+
+ for name, job in self.context.jobs.items():
+ try:
+ await job.start()
+ except Exception as e:
+ self.logger.exception(f"Failed to start job '{name}': {e}")
+
+ async def _close(self) -> None:
+ """Close all jobs, then components in reverse."""
+ for name, job in self.context.jobs.items():
+ try:
+ await job.close()
+ except Exception as e:
+ self.logger.exception(f"Failed to close job '{name}': {e}")
+
+ for components in self.context.components.values():
+ for component in components.values():
+ try:
+ await component.close()
+ except Exception as e:
+ self.logger.exception(f"Failed to close {component.component_type.value}:{component.name}: {e}")
+
+ async def run_job(self, name: str, /, **kwargs) -> Response:
+ """Execute a registered job by name."""
+ if name not in self.context.jobs:
+ raise KeyError(f"Job '{name}' not found")
+ return await self.context.jobs[name](**kwargs)
+
+ async def run_stream_job(self, name: str, /, **kwargs) -> AsyncGenerator[StreamChunk, None]:
+ """Execute a streaming job and yield chunks."""
+ if name not in self.context.jobs:
+ raise KeyError(f"Job '{name}' not found")
+ job = self.context.jobs[name]
+ stream_queue = asyncio.Queue()
+ task = asyncio.create_task(job(stream_queue=stream_queue, **kwargs))
+ async for chunk in execute_stream_task(
+ stream_queue=stream_queue,
+ task=task,
+ task_name=name,
+ output_format="chunk",
+ ):
+ assert isinstance(chunk, StreamChunk)
+ yield chunk
+
+ def run_app(self):
+ """Start the service and serve the application."""
+ if self.context.service is None:
+ raise RuntimeError("Service not configured")
+ self.context.service.run_app(app=self)
diff --git a/reme4/components/__init__.py b/reme4/components/__init__.py
new file mode 100644
index 00000000..148eeeaa
--- /dev/null
+++ b/reme4/components/__init__.py
@@ -0,0 +1,43 @@
+"""Components"""
+
+from . import as_llm
+from . import as_llm_formatter
+from . import as_token_counter
+from . import client
+from . import embedding
+from . import file_graph
+from . import file_parser
+from . import file_store
+from . import file_watcher
+from . import job
+from . import keyword_index
+from . import service
+from . import tokenizer
+from .application_context import ApplicationContext
+from .base_component import BaseComponent
+from .component_registry import ComponentRegistry, R
+from .prompt_handler import PromptHandler
+from .runtime_context import RuntimeContext
+
+__all__ = [
+ "ApplicationContext",
+ "BaseComponent",
+ "ComponentRegistry",
+ "R",
+ "PromptHandler",
+ "RuntimeContext",
+ # base components
+ "as_llm",
+ "as_llm_formatter",
+ "as_token_counter",
+ "client",
+ "embedding",
+ "file_graph",
+ "file_parser",
+ "file_store",
+ "file_watcher",
+ "job",
+ "keyword_index",
+ "service",
+ "tokenizer",
+]
diff --git a/reme4/components/application_context.py b/reme4/components/application_context.py
new file mode 100644
index 00000000..90a7f194
--- /dev/null
+++ b/reme4/components/application_context.py
@@ -0,0 +1,28 @@
+"""Application context: shared state container for components, jobs, and service."""
+
+from ..enumeration import ComponentEnum
+from ..schema import ApplicationConfig
+
+
+class ApplicationContext:
+ """Holds the parsed config and instantiated components, jobs, and service.
+
+ Acts as a passive state container. The actual wiring (resolving backends from
+ the registry and instantiating each component) is performed by Application.
+ """
+
+ def __init__(self, **kwargs):
+ # Parse and validate raw config kwargs into a typed ApplicationConfig.
+ self.app_config: ApplicationConfig = ApplicationConfig(**kwargs)
+
+ # Local imports to avoid circular dependencies during module init.
+ from .base_component import BaseComponent
+ from .job import BaseJob
+ from .service import BaseService
+
+ # Service endpoint (e.g. HTTP/MCP). Populated by Application.__init__.
+ self.service: BaseService | None = None
+ # Components keyed by type then by user-defined name.
+ self.components: dict[ComponentEnum, dict[str, BaseComponent]] = {}
+ # Jobs keyed by user-defined name.
+ self.jobs: dict[str, BaseJob] = {}
diff --git a/reme4/components/as_llm/__init__.py b/reme4/components/as_llm/__init__.py
new file mode 100644
index 00000000..82ae553f
--- /dev/null
+++ b/reme4/components/as_llm/__init__.py
@@ -0,0 +1,53 @@
+"""AgentScope LLM model wrappers."""
+
+from agentscope.model import AnthropicChatModel, ChatModelBase, OpenAIChatModel
+
+from ..base_component import BaseComponent
+from ..component_registry import R
+from ...enumeration import ComponentEnum
+
+
+class BaseAsLLM(BaseComponent):
+ """Base wrapper for AgentScope chat models. Builds ``self.model`` in ``_start``."""
+
+ component_type = ComponentEnum.AS_LLM
+
+ def __init__(self, **kwargs) -> None:
+ super().__init__(**kwargs)
+ self.model: ChatModelBase | None = None
+
+ async def _close(self) -> None:
+ self.model = None
+
+
+@R.register("openai")
+class OpenAIAsLLM(BaseAsLLM):
+ """OpenAI chat model wrapper."""
+
+ async def _start(self) -> None:
+ self.model = OpenAIChatModel(**self.kwargs)
+
+ async def _close(self) -> None:
+ if self.model is not None:
+ assert isinstance(self.model, OpenAIChatModel)
+ await self.model.client.close()
+
+
+@R.register("anthropic")
+class AnthropicAsLLM(BaseAsLLM):
+ """Anthropic chat model wrapper."""
+
+ async def _start(self) -> None:
+ self.model = AnthropicChatModel(**self.kwargs)
+
+ async def _close(self) -> None:
+ if self.model is not None:
+ assert isinstance(self.model, AnthropicChatModel)
+ await self.model.client.close()
+
+
+__all__ = [
+ "BaseAsLLM",
+ "OpenAIAsLLM",
+ "AnthropicAsLLM",
+]
diff --git a/reme4/components/as_llm_formatter/__init__.py b/reme4/components/as_llm_formatter/__init__.py
new file mode 100644
index 00000000..81e4fe37
--- /dev/null
+++ b/reme4/components/as_llm_formatter/__init__.py
@@ -0,0 +1,44 @@
+"""AgentScope LLM formatter wrappers."""
+
+from agentscope.formatter import AnthropicChatFormatter, FormatterBase
+
+from .reme_openai_chat_formatter import ReMeOpenAIChatFormatter
+from ..base_component import BaseComponent
+from ..component_registry import R
+from ...enumeration import ComponentEnum
+
+
+class BaseAsLLMFormatter(BaseComponent):
+ """Base wrapper for AgentScope formatters. Builds ``self.formatter`` in ``_start``."""
+
+ component_type = ComponentEnum.AS_LLM_FORMATTER
+
+ def __init__(self, **kwargs) -> None:
+ super().__init__(**kwargs)
+ self.formatter: FormatterBase | None = None
+
+ async def _close(self) -> None:
+ self.formatter = None
+
+
+@R.register("openai")
+class AsOpenAIChatFormatter(BaseAsLLMFormatter):
+ """OpenAI chat formatter wrapper (uses ReMe extensions)."""
+
+ async def _start(self) -> None:
+ self.formatter = ReMeOpenAIChatFormatter(**self.kwargs)
+
+
+@R.register("anthropic")
+class AsAnthropicChatFormatter(BaseAsLLMFormatter):
+ """Anthropic chat formatter wrapper."""
+
+ async def _start(self) -> None:
+ self.formatter = AnthropicChatFormatter(**self.kwargs)
+
+
+__all__ = [
+ "BaseAsLLMFormatter",
+ "AsOpenAIChatFormatter",
+ "AsAnthropicChatFormatter",
+]
diff --git a/reme4/components/as_llm_formatter/reme_openai_chat_formatter.py b/reme4/components/as_llm_formatter/reme_openai_chat_formatter.py
new file mode 100644
index 00000000..a40f977d
--- /dev/null
+++ b/reme4/components/as_llm_formatter/reme_openai_chat_formatter.py
@@ -0,0 +1,141 @@
+"""OpenAI chat formatter with ReMe extensions: image promotion and reasoning_content."""
+
+import json
+from typing import Any
+
+from agentscope.formatter import OpenAIChatFormatter
+
+# noinspection PyProtectedMember
+from agentscope.formatter._openai_formatter import (
+ _format_openai_image_block,
+ _to_openai_audio_data,
+)
+from agentscope.message import Msg, TextBlock, ImageBlock, URLSource
+
+
+def _format_openai_video_block(video_block: dict) -> dict[str, Any]:
+ """Convert a video block to OpenAI ``video_url`` content."""
+ source = video_block["source"]
+ if source["type"] == "url":
+ url = source["url"]
+ elif source["type"] == "base64":
+ url = f"data:{source['media_type']};base64,{source['data']}"
+ else:
+ raise ValueError(f"Unsupported video source type: {source['type']}")
+ return {"type": "video_url", "video_url": {"url": url}}
+
+
+class ReMeOpenAIChatFormatter(OpenAIChatFormatter):
+ """OpenAIChatFormatter + tool-result image promotion + reasoning_content passthrough."""
+
+ async def _format(self, msgs: list[Msg]) -> list[dict[str, Any]]:
+ """Format ``Msg`` list into OpenAI chat-completion message dicts."""
+ self.assert_list_of_msgs(msgs)
+
+ messages: list[dict] = []
+ i = 0
+ while i < len(msgs):
+ msg = msgs[i]
+ content_blocks = []
+ tool_calls = []
+ reasoning_content_blocks = []
+
+ for block in msg.get_content_blocks():
+ typ = block.get("type")
+
+ if typ == "text":
+ content_blocks.append({**block})
+
+ elif typ == "thinking":
+ reasoning_content_blocks.append({**block})
+
+ elif typ == "tool_use":
+ tool_calls.append(
+ {
+ "id": block.get("id"),
+ "type": "function",
+ "function": {
+ "name": block.get("name"),
+ "arguments": json.dumps(block.get("input", {}), ensure_ascii=False),
+ },
+ },
+ )
+
+ elif typ == "tool_result":
+ textual_output, multimodal_data = self.convert_tool_result_to_string(block["output"])
+ messages.append(
+ {
+ "role": "tool",
+ "tool_call_id": block.get("id"),
+ "content": textual_output,
+ "name": block.get("name"),
+ },
+ )
+
+ # OpenAI tool messages can't carry images; promote to a follow-up user message.
+ promoted_blocks = []
+ for url, multimodal_block in multimodal_data:
+ if multimodal_block["type"] == "image" and self.promote_tool_result_images:
+ promoted_blocks.extend(
+ [
+ TextBlock(type="text", text=f"\n- The image from '{url}': "),
+ ImageBlock(type="image", source=URLSource(type="url", url=url)),
+ ],
+ )
+
+ if promoted_blocks:
+ promoted_blocks = [
+ TextBlock(
+ type="text",
+ text="The following are the image contents from the tool "
+ f"result of '{block['name']}':",
+ ),
+ *promoted_blocks,
+ TextBlock(type="text", text=""),
+ ]
+ msgs.insert(
+ i + 1,
+ Msg(name="user", content=promoted_blocks, role="user"),
+ )
+
+ elif typ == "image":
+ content_blocks.append(_format_openai_image_block(block))
+
+ elif typ == "audio":
+ # Skip assistant audio — not a valid input modality.
+ if msg.role == "assistant":
+ continue
+ content_blocks.append(
+ {
+ "type": "input_audio",
+ "input_audio": _to_openai_audio_data(block["source"]),
+ },
+ )
+
+ elif typ == "video":
+ # Skip assistant video — not a valid input modality.
+ if msg.role == "assistant":
+ continue
+ content_blocks.append(_format_openai_video_block(block))
+
+ msg_openai = {
+ "role": msg.role,
+ "name": msg.name,
+ "content": content_blocks or None,
+ }
+
+ if tool_calls:
+ msg_openai["tool_calls"] = tool_calls
+
+ # Merge thinking blocks into reasoning_content for compatible models.
+ if reasoning_content_blocks:
+ reasoning_msg = "\n".join(r.get("thinking", "") for r in reasoning_content_blocks)
+ if reasoning_msg:
+ msg_openai["reasoning_content"] = reasoning_msg
+
+ if msg_openai["content"] or msg_openai.get("tool_calls"):
+ messages.append(msg_openai)
+
+ i += 1
+
+ return messages
diff --git a/reme4/components/as_token_counter/__init__.py b/reme4/components/as_token_counter/__init__.py
new file mode 100644
index 00000000..36bef1bc
--- /dev/null
+++ b/reme4/components/as_token_counter/__init__.py
@@ -0,0 +1,35 @@
+"""AgentScope token counter wrappers."""
+
+from agentscope.token import TokenCounterBase
+
+from .estimate_token_counter import EstimatedTokenCounter
+from ..base_component import BaseComponent
+from ..component_registry import R
+from ...enumeration import ComponentEnum
+
+
+class BaseAsTokenCounter(BaseComponent):
+ """Base wrapper for AgentScope token counters. Builds ``self.token_counter`` in ``_start``."""
+
+ component_type = ComponentEnum.AS_TOKEN_COUNTER
+
+ def __init__(self, **kwargs) -> None:
+ super().__init__(**kwargs)
+ self.token_counter: TokenCounterBase | None = None
+
+ async def _close(self) -> None:
+ self.token_counter = None
+
+
+@R.register("estimated")
+class EstimatedAsTokenCounter(BaseAsTokenCounter):
+ """Character-based estimated token counter — fast but approximate."""
+
+ async def _start(self) -> None:
+ self.token_counter = EstimatedTokenCounter(**self.kwargs)
+
+
+__all__ = [
+ "BaseAsTokenCounter",
+ "EstimatedAsTokenCounter",
+]
diff --git a/reme4/components/as_token_counter/estimate_token_counter.py b/reme4/components/as_token_counter/estimate_token_counter.py
new file mode 100644
index 00000000..b5566dec
--- /dev/null
+++ b/reme4/components/as_token_counter/estimate_token_counter.py
@@ -0,0 +1,21 @@
+"""Character-based token-count estimator."""
+
+from agentscope.token import TokenCounterBase
+
+
+class EstimatedTokenCounter(TokenCounterBase):
+ """Approximate token count as ``encoded_byte_len / divisor``.
+
+ Cheap proxy when exact counts aren't needed; use the model's real
+ tokenizer for accuracy.
+ """
+
+ def __init__(self, estimate_divisor: float = 4, encoding: str = "utf-8"):
+ if estimate_divisor <= 0:
+ raise ValueError("estimate_divisor must be positive")
+ self.estimate_divisor: float = estimate_divisor
+ self.encoding: str = encoding
+
+ async def count(self, text: str, **_kwargs) -> int:
+ """Estimated token count for ``text``."""
+ return int(len(text.encode(self.encoding)) / self.estimate_divisor + 0.5)
diff --git a/reme4/components/base_component.py b/reme4/components/base_component.py
new file mode 100644
index 00000000..4c614d52
--- /dev/null
+++ b/reme4/components/base_component.py
@@ -0,0 +1,185 @@
+"""Base class for components."""
+
+import asyncio
+from abc import ABC
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast
+
+from ..enumeration import ComponentEnum
+from ..utils import get_logger
+
+if TYPE_CHECKING:
+ from .application_context import ApplicationContext
+
+T = TypeVar("T", bound="BaseComponent")
+
+
+class Dependency:
+ """Declared dependency: bind() return value, instance attribute placeholder, and topological-sort edge."""
+
+ __slots__ = ("ctype", "name", "default_factory", "optional")
+
+ def __init__(
+ self,
+ ctype: ComponentEnum,
+ name: str,
+ default_factory: Callable[[], Any] | None = None,
+ optional: bool = True,
+ ) -> None:
+ self.ctype = ctype
+ self.name = name
+ self.default_factory = default_factory
+ self.optional = optional
+
+ def __repr__(self) -> str:
+ suffix = "?" if self.optional else ""
+ return f""
+
+ def __getattr__(self, item: str) -> Any:
+ # Guard against using the dependency before start() resolves it.
+ raise RuntimeError(
+ f"Dependency {self.ctype.value}:{self.name} accessed before start() (attribute '{item}')",
+ )
+
+
+class BaseComponent(ABC):
+ """Async lifecycle base class with bind-based dependency injection."""
+
+ component_type = ComponentEnum.BASE
+
+ def __init__(
+ self,
+ name: str | None = None,
+ backend: str = "",
+ app_context: "ApplicationContext | None" = None,
+ **kwargs,
+ ) -> None:
+ self.name: str = name or self.__class__.__name__
+ self.backend: str = backend
+ self.app_context: "ApplicationContext | None" = app_context
+ self.kwargs: dict = dict(kwargs)
+ self.logger = get_logger()
+ if hasattr(self.logger, "bind"):
+ self.logger = self.logger.bind(component=self.name)
+
+ self._is_started: bool = False
+ self._lock: asyncio.Lock = asyncio.Lock()
+ # Components created from bind() default_factory in standalone mode (auto-managed lifecycle).
+ self._owned: list["BaseComponent"] = []
+
+ @property
+ def is_started(self) -> bool:
+ """Whether the component has been started."""
+ return self._is_started
+
+ # ----- Dependency declaration ----------------------------------------
+
+ @staticmethod
+ def bind(
+ name: str | None,
+ base_cls: type[T],
+ *,
+ default_factory: Callable[[], T] | None = None,
+ optional: bool = True,
+ ) -> T | None:
+ """Declare a dependency on another component; resolved at start(). Empty name → None."""
+ if not name:
+ return None
+ ctype = getattr(base_cls, "component_type", None)
+ if not isinstance(ctype, ComponentEnum) or ctype is ComponentEnum.BASE:
+ raise TypeError(f"{base_cls.__name__} must declare a non-BASE ComponentEnum 'component_type'")
+ return cast(T, Dependency(ctype, name, default_factory, optional))
+
+ @property
+ def dependencies(self) -> list[Dependency]:
+ """All unresolved bindings declared on this instance."""
+ return [v for v in self.__dict__.values() if isinstance(v, Dependency)]
+
+ async def _resolve_bindings(self) -> None:
+ """Replace Dependency placeholders with real components (or default_factory / None for optional)."""
+ for attr, value in list(self.__dict__.items()):
+ if not isinstance(value, Dependency):
+ continue
+ if self.app_context is None:
+ # Standalone mode: factory or (optional → None) or keep placeholder.
+ if value.default_factory is not None:
+ instance = value.default_factory()
+ setattr(self, attr, instance)
+ if isinstance(instance, BaseComponent):
+ self._owned.append(instance)
+ elif value.optional:
+ setattr(self, attr, None)
+ else:
+ target = self.app_context.components.get(value.ctype, {}).get(value.name)
+ if target is not None:
+ setattr(self, attr, target)
+ elif value.optional:
+ setattr(self, attr, None)
+ else:
+ raise ValueError(f"{value.ctype.value} '{value.name}' not found.")
+
+ # ----- Lookup --------------------------------------------------------
+
+ @property
+ def working_path(self) -> Path:
+ """Resolved working directory from app context or cwd."""
+ if self.app_context is None:
+ return Path.cwd()
+ return Path(self.app_context.app_config.working_dir)
+
+ @property
+ def working_metadata_path(self) -> Path:
+ """Resolved metadata directory: working_path / metadata_dir, or absolute metadata_dir."""
+ if self.app_context is None:
+ return Path.cwd() / "metadata"
+ return self.working_path / self.app_context.app_config.metadata_dir
+
+ # ----- Lifecycle -----------------------------------------------------
+
+ async def _start(self) -> None:
+ """Subclass hook: start logic."""
+
+ async def _close(self) -> None:
+ """Subclass hook: close logic."""
+
+ async def dump(self) -> None:
+ """Persist in-memory state to disk. Override in subclasses that need persistence."""
+
+ async def load(self) -> None:
+ """Restore in-memory state from disk. Override in subclasses that need persistence."""
+
+ async def start(self) -> None:
+ """Resolve bindings → start owned fallbacks → _start(). No-op if already started."""
+ async with self._lock:
+ if self._is_started:
+ return
+ await self._resolve_bindings()
+ for owned in self._owned:
+ await owned.start()
+ await self._start()
+ self._is_started = True
+
+ async def close(self) -> None:
+ """_close() → close owned fallbacks in reverse. No-op if not started."""
+ async with self._lock:
+ if not self._is_started:
+ return
+ await self._close()
+ for owned in reversed(self._owned):
+ await owned.close()
+ self._is_started = False
+
+ async def restart(self) -> None:
+ """Close then start."""
+ await self.close()
+ await self.start()
+
+ async def __call__(self, **kwargs):
+ raise NotImplementedError
+
+ async def __aenter__(self) -> "BaseComponent":
+ await self.start()
+ return self
+
+ async def __aexit__(self, exc_type, exc_val, exc_tb) -> None:
+ await self.close()
diff --git a/reme4/components/client/__init__.py b/reme4/components/client/__init__.py
new file mode 100644
index 00000000..5c212bac
--- /dev/null
+++ b/reme4/components/client/__init__.py
@@ -0,0 +1,7 @@
+"""Client components."""
+
+from .base_client import BaseClient
+from .http_client import HttpClient
+from .mcp_client import MCPClient
+
+__all__ = ["BaseClient", "HttpClient", "MCPClient"]
diff --git a/reme4/components/client/base_client.py b/reme4/components/client/base_client.py
new file mode 100644
index 00000000..a5b05b6c
--- /dev/null
+++ b/reme4/components/client/base_client.py
@@ -0,0 +1,41 @@
+"""Base client abstraction."""
+
+import json
+from abc import abstractmethod
+from collections.abc import AsyncGenerator
+
+from ..base_component import BaseComponent
+from ...enumeration import ComponentEnum
+
+
+class BaseClient(BaseComponent):
+ """Abstract base for clients that communicate with ReMe services."""
+
+ component_type = ComponentEnum.CLIENT
+
+ def __init__(self, **kwargs) -> None:
+ super().__init__(**kwargs)
+ self.client = None
+
+ async def _start(self) -> None:
+ """Initialize the client."""
+
+ async def _close(self) -> None:
+ """Close the client and release resources."""
+
+ @abstractmethod
+ def _execute(self) -> AsyncGenerator[str, None]:
+ """Backend-specific execution; yield text chunks (single yield for non-streaming backends)."""
+
+ @abstractmethod
+ async def list_actions(self) -> list[dict]:
+ """Discover available actions on the server; each dict is the raw backend descriptor."""
+
+ async def __call__(self) -> AsyncGenerator[str, None]:
+ """Dispatch: action='list' returns the action catalog; otherwise delegate to _execute()."""
+ if getattr(self, "action", None) == "list":
+ actions = await self.list_actions()
+ yield json.dumps(actions, indent=2, ensure_ascii=False)
+ return
+ async for chunk in self._execute():
+ yield chunk
diff --git a/reme4/components/client/http_client.py b/reme4/components/client/http_client.py
new file mode 100644
index 00000000..efec1ff7
--- /dev/null
+++ b/reme4/components/client/http_client.py
@@ -0,0 +1,143 @@
+"""HTTP client for ReMe services."""
+
+import json
+import os
+from collections.abc import AsyncGenerator
+
+import httpx
+
+from .base_client import BaseClient
+from ..component_registry import R
+from ...constants import REME_SERVICE_INFO, REME_DEFAULT_HOST, REME_DEFAULT_PORT
+from ...enumeration import ChunkEnum
+from ...schema import StreamChunk
+
+
+@R.register("http")
+class HttpClient(BaseClient):
+ """HTTP client that auto-adapts to JSON or SSE endpoints via Content-Type."""
+
+ def __init__(
+ self,
+ action: str,
+ host: str | None = None,
+ port: int | None = None,
+ timeout: float = 30.0,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+
+ # Resolve host/port: explicit args > env var > defaults
+ if not (host and port):
+ if service_info := os.environ.get(REME_SERVICE_INFO):
+ try:
+ data = json.loads(service_info)
+ host = data["host"]
+ port = data["port"]
+ except Exception:
+ self.logger.warning(f"Invalid service info: {service_info}")
+ host, port = REME_DEFAULT_HOST, REME_DEFAULT_PORT
+ else:
+ host, port = REME_DEFAULT_HOST, REME_DEFAULT_PORT
+
+ self.action = action
+ self.base_url = f"http://{host}:{port}"
+ self.timeout = timeout
+
+ async def _start(self) -> None:
+ """Initialize the HTTP client."""
+ if self.client is None:
+ self.client = httpx.AsyncClient(base_url=self.base_url, timeout=self.timeout)
+
+ async def _iter_stream_chunks(self) -> AsyncGenerator[StreamChunk, None]:
+ """Send request and yield raw StreamChunks; auto-detects JSON vs SSE via Content-Type.
+
+ For JSON responses: yields a single CONTENT chunk with the raw response body.
+ For SSE responses: yields each streaming chunk as it arrives.
+ """
+ if self.client is None:
+ raise RuntimeError("Client not initialized. Call _start() first.")
+
+ async with self.client.stream("POST", f"/{self.action}", json=self.kwargs) as resp:
+ resp.raise_for_status()
+ ctype = resp.headers.get("content-type", "")
+
+ if ctype.startswith("text/event-stream"):
+ async for line in resp.aiter_lines():
+ if not line.startswith("data:"):
+ continue
+ payload = line[len("data:") :]
+ if payload.strip() == "[DONE]":
+ return
+ try:
+ data = json.loads(payload)
+ except json.JSONDecodeError:
+ continue
+ chunk = StreamChunk(**data)
+ if chunk.chunk_type == ChunkEnum.ERROR:
+ # Surface server-side errors as exceptions so callers don't
+ # mistake error chunks for valid content.
+ raise RuntimeError(str(chunk.chunk))
+ if chunk.done:
+ return
+ yield chunk
+ else:
+ body = await resp.aread()
+ yield StreamChunk(chunk_type=ChunkEnum.CONTENT, chunk=body.decode())
+
+ async def stream_chunks(self) -> AsyncGenerator[StreamChunk, None]:
+ """HTTP-specific richer access: yield raw StreamChunk objects (no display formatting)."""
+ async for chunk in self._iter_stream_chunks():
+ yield chunk
+
+ async def list_actions(self) -> list[dict]:
+ """Return raw OpenAPI operations; each dict gets an `action` key (path without leading '/')."""
+ if self.client is None:
+ raise RuntimeError("Client not initialized. Call _start() first.")
+ resp = await self.client.get("/openapi.json")
+ resp.raise_for_status()
+ spec = resp.json()
+ actions: list[dict] = []
+ for path, methods in spec.get("paths", {}).items():
+ for method, op in methods.items():
+ actions.append({"action": path.lstrip("/"), "method": method.upper(), **op})
+ return actions
+
+ @staticmethod
+ def _format_for_display(text: str) -> str:
+ """Render a JSON response as human-friendly CLI text; pass through unrecognized payloads."""
+ try:
+ data = json.loads(text)
+ except (ValueError, json.JSONDecodeError):
+ return text
+ if not (isinstance(data, dict) and isinstance(data.get("answer"), str)):
+ return json.dumps(data, indent=2, ensure_ascii=False) if isinstance(data, (dict, list)) else text
+ d = dict(data)
+ answer = d.pop("answer")
+ success = d.pop("success", None)
+ metadata = d.pop("metadata", None)
+ parts = [answer]
+ status_pieces = []
+ if success is not None:
+ status_pieces.append("✅" if success else "❌")
+ if metadata:
+ status_pieces.append(json.dumps(metadata, ensure_ascii=False))
+ if status_pieces:
+ parts.append(" ".join(status_pieces))
+ if d:
+ parts.append(json.dumps(d, indent=2, ensure_ascii=False))
+ return "\n".join(parts)
+
+ # pylint: disable=invalid-overridden-method
+ async def _execute(self) -> AsyncGenerator[str, None]:
+ """Yield text chunks for CLI display; JSON responses are pretty-formatted."""
+ async for chunk in self._iter_stream_chunks():
+ payload = chunk.chunk
+ text = payload if isinstance(payload, str) else json.dumps(payload, ensure_ascii=False)
+ yield self._format_for_display(text)
+
+ async def _close(self) -> None:
+ """Close the HTTP client."""
+ if self.client is not None:
+ await self.client.aclose()
+ self.client = None
diff --git a/reme4/components/client/mcp_client.py b/reme4/components/client/mcp_client.py
new file mode 100644
index 00000000..b8b17a9b
--- /dev/null
+++ b/reme4/components/client/mcp_client.py
@@ -0,0 +1,125 @@
+"""MCP client for ReMe services."""
+
+import json
+import os
+from collections.abc import AsyncGenerator
+from typing import Any
+
+from fastmcp import Client
+from fastmcp.client import SSETransport, StdioTransport, StreamableHttpTransport
+from fastmcp.client.client import CallToolResult
+
+from .base_client import BaseClient
+from ..component_registry import R
+from ...constants import REME_SERVICE_INFO, REME_DEFAULT_HOST, REME_DEFAULT_PORT
+
+_TRANSPORT_MAP = {
+ "sse": SSETransport,
+ "stdio": StdioTransport,
+ "streamable-http": StreamableHttpTransport,
+}
+
+
+@R.register("mcp")
+class MCPClient(BaseClient):
+ """MCP client that communicates with ReMe MCP service via fastmcp.Client.
+
+ Usage:
+ # SSE (default)
+ client = MCPClient(action="my_tool", host="localhost", port=8000, query="hello")
+ async with client:
+ async for text in client():
+ print(text)
+
+ # Streamable HTTP
+ client = MCPClient(action="my_tool", transport="streamable-http", host="localhost", port=8000)
+
+ # Stdio
+ client = MCPClient(action="my_tool", transport="stdio", command="python", args=["server.py"])
+
+ # Custom transport object
+ from fastmcp.client import SSETransport
+ client = MCPClient(action="my_tool", transport=SSETransport(url="http://host:port/sse"))
+ """
+
+ def __init__(
+ self,
+ action: str,
+ transport: str | Any = "sse",
+ host: str | None = None,
+ port: int | None = None,
+ timeout: float = 30.0,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+
+ if isinstance(transport, str) and transport not in _TRANSPORT_MAP:
+ raise ValueError(f"Unknown transport: {transport!r}, expected one of {list(_TRANSPORT_MAP)}")
+
+ if isinstance(transport, str) and transport != "stdio":
+ if not (host and port):
+ if service_info := os.environ.get(REME_SERVICE_INFO):
+ try:
+ data = json.loads(service_info)
+ host = data["host"]
+ port = data["port"]
+ except Exception:
+ self.logger.warning(f"Invalid service info: {service_info}")
+ host, port = REME_DEFAULT_HOST, REME_DEFAULT_PORT
+ else:
+ host, port = REME_DEFAULT_HOST, REME_DEFAULT_PORT
+ self.host = host
+ self.port = port
+
+ self.action = action
+ self.transport = transport
+ self.timeout = timeout
+
+ def _build_transport(self):
+ if not isinstance(self.transport, str):
+ return self.transport
+
+ cls = _TRANSPORT_MAP[self.transport]
+
+ if self.transport == "stdio":
+ command = self.kwargs.pop("command", "")
+ args = self.kwargs.pop("args", [])
+ return cls(command=command, args=args)
+
+ path = "/sse" if self.transport == "sse" else "/mcp"
+ url = f"http://{self.host}:{self.port}{path}"
+ return cls(url=url)
+
+ # pylint: disable=unnecessary-dunder-call
+ async def _start(self) -> None:
+ if self.client is None:
+ self.client = Client(self._build_transport(), timeout=self.timeout)
+ await self.client.__aenter__()
+
+ # pylint: disable=invalid-overridden-method
+ async def _execute(self) -> AsyncGenerator[str, None]:
+ if self.client is None:
+ raise RuntimeError("Client not initialized. Call _start() first.")
+
+ result: CallToolResult = await self.client.call_tool(self.action, self.kwargs)
+ yield self._extract_text(result)
+
+ async def list_actions(self) -> list[dict]:
+ """Return raw MCP Tool dumps; each dict gets an `action` key (the tool name)."""
+ if self.client is None:
+ raise RuntimeError("Client not initialized. Call _start() first.")
+ tools = await self.client.list_tools()
+ return [tool.model_dump() for tool in tools]
+
+ # pylint: disable=unnecessary-dunder-call
+ async def _close(self) -> None:
+ if self.client is not None:
+ await self.client.__aexit__(None, None, None)
+ self.client = None
+
+ @staticmethod
+ def _extract_text(result: CallToolResult) -> str:
+ for block in result.content:
+ if hasattr(block, "text"):
+ return block.text
+ return str(result.content)
diff --git a/reme4/components/component_registry.py b/reme4/components/component_registry.py
new file mode 100644
index 00000000..c78ceb9c
--- /dev/null
+++ b/reme4/components/component_registry.py
@@ -0,0 +1,77 @@
+"""Global registry mapping (ComponentEnum, name) -> component class."""
+
+from typing import Callable, TypeVar, cast
+
+from .base_component import BaseComponent
+from ..enumeration import ComponentEnum
+from ..utils import get_logger
+
+T = TypeVar("T", bound=BaseComponent)
+
+
+class ComponentRegistry:
+ """Two-level registry: component_type -> name -> class.
+
+ Supports both direct calls — ``R.register(MyClass, "name")`` — and
+ decorator usage — ``@R.register("name")``.
+ """
+
+ def __init__(self) -> None:
+ self._registry: dict[ComponentEnum, dict[str, type[BaseComponent]]] = {}
+ self.logger = get_logger()
+
+ def _do_register(self, cls: type[T], name: str) -> type[T]:
+ """Insert `cls` under its `component_type` group; warn on overwrite."""
+ component_type = getattr(cls, "component_type", None)
+ if not isinstance(component_type, ComponentEnum):
+ raise TypeError(f"{cls.__name__} must have a ComponentEnum 'component_type' attribute")
+ if not name:
+ raise ValueError("Component name cannot be empty")
+
+ group = self._registry.setdefault(component_type, {})
+ if name in group:
+ self.logger.warning(f"Component '{name}' already registered for {component_type}, overwriting")
+ group[name] = cls
+ return cls
+
+ def register(
+ self,
+ cls_or_name: type[T] | str,
+ name: str | None = None,
+ ) -> Callable[[type[T]], type[T]] | type[T]:
+ """Register a component class directly, or return a decorator that does so."""
+ # Direct mode: first arg is the class itself.
+ if isinstance(cls_or_name, type):
+ return self._do_register(cast(type[T], cls_or_name), name if name is not None else cls_or_name.__name__)
+
+ # Decorator mode: first arg is the registration name.
+ if not isinstance(cls_or_name, str):
+ raise TypeError(f"Expected a class or string, got {type(cls_or_name).__name__}")
+
+ def decorator(decorated_cls: type[T]) -> type[T]:
+ return self._do_register(decorated_cls, cls_or_name)
+
+ return decorator
+
+ def get(self, component_type: ComponentEnum, name: str) -> type[BaseComponent] | None:
+ """Look up a registered class; return None if not found."""
+ return self._registry.get(component_type, {}).get(name)
+
+ def get_all(self, component_type: ComponentEnum) -> dict[str, type[BaseComponent]]:
+ """Return a shallow copy of all classes registered under `component_type`."""
+ return dict(self._registry.get(component_type, {}))
+
+ def unregister(self, component_type: ComponentEnum, name: str) -> bool:
+ """Remove an entry; return True if it existed, False otherwise."""
+ if (group := self._registry.get(component_type)) and name in group:
+ del group[name]
+ return True
+ return False
+
+ def clear(self) -> None:
+ """Drop every registered entry."""
+ self._registry.clear()
+
+
+# Process-wide singleton used throughout the codebase.
+R = ComponentRegistry()
diff --git a/reme4/components/embedding/__init__.py b/reme4/components/embedding/__init__.py
new file mode 100644
index 00000000..bf2d4b48
--- /dev/null
+++ b/reme4/components/embedding/__init__.py
@@ -0,0 +1,6 @@
+"""Embedding model implementations."""
+
+from .base_embedding_model import BaseEmbeddingModel
+from .openai_embedding_model import OpenAIEmbeddingModel
+
+__all__ = ["BaseEmbeddingModel", "OpenAIEmbeddingModel"]
diff --git a/reme4/components/embedding/base_embedding_model.py b/reme4/components/embedding/base_embedding_model.py
new file mode 100644
index 00000000..0d03ec61
--- /dev/null
+++ b/reme4/components/embedding/base_embedding_model.py
@@ -0,0 +1,214 @@
+"""Base embedding model with LRU cache and disk persistence."""
+
+import asyncio
+import hashlib
+import os
+from abc import abstractmethod
+from collections import OrderedDict
+from pathlib import Path
+
+import numpy as np
+
+from ..base_component import BaseComponent
+from ...enumeration import ComponentEnum
+from ...schema import EmbNode
+
+
+class BaseEmbeddingModel(BaseComponent):
+ """Embedding model with LRU cache and disk persistence."""
+
+ component_type = ComponentEnum.EMBEDDING_MODEL
+
+ def __init__(
+ self,
+ api_key: str | None = None,
+ base_url: str | None = None,
+ model_name: str = "",
+ dimensions: int = 1024,
+ pass_dimensions: bool = False,
+ max_batch_size: int = 10,
+ max_input_length: int = 8192,
+ max_cache_size: int = 10000,
+ enable_cache: bool = True,
+ cache_version: str = "v1",
+ max_retries: int = 3,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ self.api_key = api_key or os.environ.get("EMBEDDING_API_KEY", "")
+ self.base_url = base_url or os.environ.get("EMBEDDING_BASE_URL", "")
+ self.model_name = model_name
+ self.dimensions = dimensions
+ self.pass_dimensions = pass_dimensions
+ self.max_batch_size = max_batch_size
+ self.max_input_length = max_input_length
+ self.max_cache_size = max_cache_size
+ self.enable_cache = enable_cache
+ self.cache_version = cache_version
+ self.max_retries = max_retries
+ self._embedding_cache: OrderedDict[str, np.ndarray] = OrderedDict()
+ self.is_healthy: bool = True
+
+ @property
+ def cache_path(self) -> Path:
+ """Disk path for the embedding cache file."""
+ return self.working_metadata_path / "embedding_cache" / f"{self.name}_{self.cache_version}.npz"
+
+ async def _start(self) -> None:
+ """Load cache from disk on startup."""
+ await self.load()
+
+ async def health_check(self, timeout: float = 2.0) -> bool:
+ """Probe the provider; sets and returns is_healthy."""
+ tag = f"[EMBEDDING HEALTH CHECK] name={self.name} model={self.model_name}"
+ try:
+ result = await asyncio.wait_for(self._get_embeddings(["ping"]), timeout=timeout)
+ if not result or result[0] is None:
+ raise RuntimeError("empty embedding")
+ self.is_healthy = True
+ self.logger.info(f"{tag} -> OK")
+ except asyncio.TimeoutError:
+ self.is_healthy = False
+ self.logger.error(f"{tag} -> FAIL timeout({timeout}s)")
+ except Exception as e:
+ self.is_healthy = False
+ self.logger.error(f"{tag} -> FAIL {type(e).__name__}: {e}")
+ return self.is_healthy
+
+ async def _close(self) -> None:
+ """Persist cache to disk on shutdown."""
+ await self.dump()
+
+ # -- Public API --
+
+ async def get_embedding(self, input_text: str, **kwargs) -> np.ndarray | None:
+ """Get embedding for a single text."""
+ results = await self.get_embeddings([input_text], **kwargs)
+ return results[0] if results else None
+
+ async def get_embeddings(self, input_text: list[str], **kwargs) -> list[np.ndarray | None]:
+ """Get embeddings for a list of texts, with caching and batching."""
+ truncated = [t[: self.max_input_length] for t in input_text]
+ results: list[np.ndarray | None] = [None] * len(truncated)
+ to_compute: list[tuple[int, str]] = []
+
+ # Split into cache hits and misses
+ for idx, text in enumerate(truncated):
+ cached = self._get_from_cache(text)
+ if cached is not None:
+ results[idx] = cached
+ else:
+ to_compute.append((idx, text))
+
+ # Batch-compute misses with retry
+ if to_compute:
+ for i in range(0, len(to_compute), self.max_batch_size):
+ batch = to_compute[i : i + self.max_batch_size]
+ indices = [idx for idx, _ in batch]
+ texts = [text for _, text in batch]
+
+ embeddings = None
+ for attempt in range(self.max_retries):
+ try:
+ embeddings = await self._get_embeddings(texts, **kwargs)
+ if embeddings and len(embeddings) == len(texts):
+ break
+ except (TimeoutError, ConnectionError, OSError):
+ if attempt < self.max_retries - 1:
+ await asyncio.sleep(2**attempt)
+ except Exception:
+ self.logger.exception("Embedding request failed")
+ break
+
+ if not embeddings or len(embeddings) != len(texts):
+ continue
+
+ # Normalize dimensions and cache
+ for orig_idx, text, emb in zip(indices, texts, embeddings):
+ if emb is None:
+ continue
+ emb_array = np.asarray(emb, dtype=np.float16)
+ if len(emb_array) != self.dimensions:
+ if len(emb_array) < self.dimensions:
+ emb_array = np.pad(emb_array, (0, self.dimensions - len(emb_array)))
+ else:
+ emb_array = emb_array[: self.dimensions]
+ results[orig_idx] = emb_array
+ self._put_to_cache(text, emb_array)
+
+ return results
+
+ async def get_node_embeddings(self, nodes: list[EmbNode], **kwargs) -> list[EmbNode]:
+ """Compute and assign embeddings for EmbNode objects."""
+ embeddings = await self.get_embeddings([n.text for n in nodes], **kwargs)
+ if len(embeddings) == len(nodes):
+ for node, vec in zip(nodes, embeddings):
+ if vec is not None:
+ node.embedding = vec
+ return nodes
+
+ @abstractmethod
+ async def _get_embeddings(self, input_text: list[str], **kwargs) -> list[list[float] | None]:
+ """Get raw embeddings from the underlying provider."""
+
+ # -- Cache Operations --
+
+ def _get_from_cache(self, text: str) -> np.ndarray | None:
+ """Lookup text in LRU cache, promoting on hit."""
+ if not self.enable_cache:
+ return None
+ key = self._get_cache_key(text)
+ if key not in self._embedding_cache:
+ return None
+ self._embedding_cache.move_to_end(key)
+ return self._embedding_cache[key]
+
+ def _put_to_cache(self, text: str, embedding: np.ndarray) -> None:
+ """Insert into LRU cache, evicting oldest if full."""
+ if not self.enable_cache or self.max_cache_size <= 0 or len(embedding) != self.dimensions:
+ return
+ key = self._get_cache_key(text)
+ if len(self._embedding_cache) >= self.max_cache_size and key not in self._embedding_cache:
+ self._embedding_cache.popitem(last=False)
+ self._embedding_cache[key] = embedding
+ self._embedding_cache.move_to_end(key)
+
+ def _get_cache_key(self, text: str) -> str:
+ """Generate cache key from text, model name, and dimensions."""
+ return hashlib.sha256(f"{text}|{self.model_name}|{self.dimensions}".encode()).hexdigest()
+
+ # -- Cache Persistence --
+
+ async def load(self) -> None:
+ """Load cached embeddings from disk (npz format); replaces in-memory cache."""
+ self._embedding_cache.clear()
+ if not self.enable_cache or not self.cache_path.exists():
+ return
+
+ try:
+ data = np.load(self.cache_path)
+ except Exception:
+ self.logger.exception("Failed to load embedding cache, removing")
+ self.cache_path.unlink(missing_ok=True)
+ return
+
+ for key, emb in zip(data["keys"], data["embeddings"]):
+ if len(emb) != self.dimensions:
+ continue
+ if len(self._embedding_cache) >= self.max_cache_size:
+ break
+ self._embedding_cache[str(key)] = emb.astype(np.float16)
+ self.logger.info(f"Loaded {len(self._embedding_cache)} embeddings from {self.cache_path}")
+
+ async def dump(self) -> None:
+ """Persist in-memory cache to disk (npz format)."""
+ if not self.enable_cache or not self._embedding_cache:
+ return
+ self.cache_path.parent.mkdir(parents=True, exist_ok=True)
+ keys = list(self._embedding_cache.keys())
+ embeddings = np.stack(list(self._embedding_cache.values()))
+ try:
+ np.savez(self.cache_path, keys=np.array(keys, dtype=str), embeddings=embeddings)
+ self.logger.info(f"Saved {len(self._embedding_cache)} embeddings to {self.cache_path}")
+ except Exception:
+ self.logger.exception("Failed to save embedding cache")
diff --git a/reme4/components/embedding/openai_embedding_model.py b/reme4/components/embedding/openai_embedding_model.py
new file mode 100644
index 00000000..a03c4a01
--- /dev/null
+++ b/reme4/components/embedding/openai_embedding_model.py
@@ -0,0 +1,52 @@
+"""OpenAI-compatible async embedding model."""
+
+from openai import AsyncOpenAI
+
+from .base_embedding_model import BaseEmbeddingModel
+from ..component_registry import R
+
+
+@R.register("openai")
+class OpenAIEmbeddingModel(BaseEmbeddingModel):
+ """Embedding model backed by any OpenAI-compatible API."""
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ self._client: AsyncOpenAI | None = None
+
+ async def _start(self) -> None:
+ """Initialize async OpenAI client."""
+ self._client = AsyncOpenAI(api_key=self.api_key, base_url=self.base_url, **self.kwargs)
+ await super()._start()
+
+ async def _close(self) -> None:
+ """Close the async OpenAI client."""
+ if self._client:
+ await self._client.close()
+ self._client = None
+ await super()._close()
+
+ async def _get_embeddings(self, input_text: list[str], **kwargs) -> list[list[float] | None]:
+ """Call the embeddings API and return results aligned to input order."""
+ if self._client is None:
+ raise RuntimeError("Client not initialized. Call _start() first.")
+
+ create_kwargs: dict = {"model": self.model_name, "input": input_text, **kwargs}
+ if self.pass_dimensions:
+ create_kwargs["dimensions"] = self.dimensions
+
+ completion = await self._client.embeddings.create(**create_kwargs)
+
+ # Map API results back to input order
+ result: list[list[float] | None] = [None] * len(input_text)
+ for emb in completion.data:
+ if 0 <= emb.index < len(input_text):
+ vec = emb.embedding or getattr(emb, "dense_embedding", None)
+ if vec is not None:
+ result[emb.index] = list(vec)
+ else:
+ self.logger.warning(f"Empty embedding at index {emb.index}")
+ else:
+ self.logger.warning(f"Index {emb.index} out of range for input length {len(input_text)}")
+
+ return result
diff --git a/reme4/components/file_graph/__init__.py b/reme4/components/file_graph/__init__.py
new file mode 100644
index 00000000..12b01355
--- /dev/null
+++ b/reme4/components/file_graph/__init__.py
@@ -0,0 +1,7 @@
+"""File graph module."""
+
+from .base_file_graph import BaseFileGraph
+from .local_file_graph import LocalFileGraph
+from .nx_file_graph import NxFileGraph
+
+__all__ = ["BaseFileGraph", "LocalFileGraph", "NxFileGraph"]
diff --git a/reme4/components/file_graph/base_file_graph.py b/reme4/components/file_graph/base_file_graph.py
new file mode 100644
index 00000000..333d75db
--- /dev/null
+++ b/reme4/components/file_graph/base_file_graph.py
@@ -0,0 +1,53 @@
+"""Abstract base for file-graph backends."""
+
+from abc import abstractmethod
+from pathlib import Path
+
+from ..base_component import BaseComponent
+from ...enumeration import ComponentEnum
+from ...schema import FileLink, FileNode
+
+
+class BaseFileGraph(BaseComponent):
+ """Abstract base for file-graph backends."""
+
+ component_type = ComponentEnum.FILE_GRAPH
+
+ def __init__(self, graph_name: str = "default", graph_version: str = "v1", **kwargs):
+ super().__init__(**kwargs)
+ self.graph_name: str = graph_name or self.name
+ self.graph_version: str = graph_version
+ self.graph_path: Path = self.working_metadata_path / self.component_type.value
+ self.graph_path.mkdir(parents=True, exist_ok=True)
+
+ # -- Node CRUD ---------------------------------------------------------
+
+ @abstractmethod
+ async def upsert_nodes(self, nodes: list[FileNode]) -> None:
+ """Insert or update nodes in the graph."""
+
+ @abstractmethod
+ async def delete_nodes(self, paths: list[str]) -> None:
+ """Delete nodes by path."""
+
+ @abstractmethod
+ async def get_nodes(self, paths: list[str] | None = None) -> list[FileNode]:
+ """Return nodes by paths; None = all real nodes; [] = []."""
+
+ @abstractmethod
+ async def rebuild_links(self) -> None:
+ """Rebuild all edges from each node's link payload."""
+
+ @abstractmethod
+ async def clear(self):
+ """Remove all nodes and edges."""
+
+ # -- Link access -------------------------------------------------------
+
+ @abstractmethod
+ async def get_outlinks(self, path: str) -> list[FileLink]:
+ """Return outgoing links for *path*."""
+
+ @abstractmethod
+ async def get_inlinks(self, path: str) -> list[FileLink]:
+ """Return incoming links for *path*."""
diff --git a/reme4/components/file_graph/local_file_graph.py b/reme4/components/file_graph/local_file_graph.py
new file mode 100644
index 00000000..e68ef46f
--- /dev/null
+++ b/reme4/components/file_graph/local_file_graph.py
@@ -0,0 +1,138 @@
+"""Pure-Python file-graph backend (no external deps)."""
+
+from pathlib import Path
+
+from .base_file_graph import BaseFileGraph
+from ..component_registry import R
+from ...schema import FileLink, FileNode
+
+
+@R.register("local")
+class LocalFileGraph(BaseFileGraph):
+ """Dict-backed file graph; uses FileLink.target_path for adjacency."""
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ self._nodes: dict[str, FileNode] = {}
+ self._inverse: dict[str, set[str]] = {} # target → {sources}
+ self._pending: dict[str, set[str]] = {} # virtual target → {sources}
+ self._graph_file: Path = self.graph_path / f"{self.graph_name}_{self.graph_version}.jsonl"
+
+ # -- Lifecycle ---------------------------------------------------------
+
+ async def _start(self) -> None:
+ await super()._start()
+ await self.load()
+ await self.rebuild_links()
+
+ async def _close(self) -> None:
+ await self.dump()
+ await super()._close()
+
+ async def load(self) -> None:
+ """Load nodes from JSONL file into memory; keep current state on failure."""
+ if not self._graph_file.exists():
+ return
+ try:
+ with open(self._graph_file, "r", encoding="utf-8") as f:
+ self._nodes.update(
+ (n.path, n) for line in f if line.strip() for n in [FileNode.model_validate_json(line)]
+ )
+ self.logger.info(f"Loaded {len(self._nodes)} nodes from {self._graph_file}")
+ except Exception as e:
+ self.logger.exception(f"Failed to load {self._graph_file}: {e}")
+
+ async def dump(self) -> None:
+ """Persist all nodes to JSONL via atomic rename."""
+ try:
+ tmp = self._graph_file.with_suffix(".tmp")
+ with open(tmp, "w", encoding="utf-8") as f:
+ f.writelines(f"{n.model_dump_json()}\n" for n in self._nodes.values())
+ tmp.replace(self._graph_file)
+ self.logger.info(f"Saved {len(self._nodes)} nodes to {self._graph_file}")
+ except Exception as e:
+ self.logger.exception(f"Failed to write {self._graph_file}: {e}")
+
+ # -- Edge bookkeeping --------------------------------------------------
+
+ def _add_edge(self, src: str, target: str) -> None:
+ """Register src→target; route to pending if target is virtual."""
+ bucket = self._inverse if target in self._nodes else self._pending
+ bucket.setdefault(target, set()).add(src)
+
+ def _remove_edge(self, src: str, target: str) -> None:
+ """Remove src→target from both inverse and pending buckets."""
+ for bucket in (self._inverse, self._pending):
+ srcs = bucket.get(target)
+ if srcs is None or src not in srcs:
+ continue
+ srcs.discard(src)
+ if not srcs:
+ del bucket[target]
+
+ # -- Node CRUD ---------------------------------------------------------
+
+ async def upsert_nodes(self, nodes: list[FileNode]) -> None:
+ for node in nodes:
+ path = node.path
+ old = self._nodes.get(path)
+ if old is not None:
+ for link in old.links:
+ if link.target_path:
+ self._remove_edge(path, link.target_path)
+ self._nodes[path] = node
+ for link in node.links:
+ if link.target_path:
+ self._add_edge(path, link.target_path)
+ # Promote pending edges that now target a real node.
+ promoted = self._pending.pop(path, None)
+ if promoted:
+ self._inverse.setdefault(path, set()).update(promoted)
+
+ async def delete_nodes(self, paths: list[str]) -> None:
+ for path in paths:
+ node = self._nodes.pop(path, None)
+ if node is None:
+ continue
+ for link in node.links:
+ if link.target_path:
+ self._remove_edge(path, link.target_path)
+ # Demote inbound edges to pending (sources still reference this path).
+ demoted = self._inverse.pop(path, None)
+ if demoted:
+ self._pending.setdefault(path, set()).update(demoted)
+
+ async def get_nodes(self, paths: list[str] | None = None) -> list[FileNode]:
+ if paths is None:
+ return list(self._nodes.values())
+ return [self._nodes[p] for p in paths if p in self._nodes]
+
+ async def rebuild_links(self) -> None:
+ """Rebuild inverse/pending indexes from all node link payloads."""
+ self._inverse.clear()
+ self._pending.clear()
+ for src, node in self._nodes.items():
+ for link in node.links:
+ if link.target_path:
+ self._add_edge(src, link.target_path)
+
+ async def clear(self):
+ self._nodes.clear()
+ self._inverse.clear()
+ self._pending.clear()
+ self._graph_file.unlink(missing_ok=True)
+
+ # -- Link access -------------------------------------------------------
+
+ async def get_outlinks(self, path: str) -> list[FileLink]:
+ node = self._nodes.get(path)
+ if node is None:
+ return []
+ return [lnk for lnk in node.links if lnk.target_path and lnk.target_path in self._nodes]
+
+ async def get_inlinks(self, path: str) -> list[FileLink]:
+ if path not in self._nodes:
+ return []
+ return [
+ link for src in self._inverse.get(path, ()) for link in self._nodes[src].links if link.target_path == path
+ ]
diff --git a/reme4/components/file_graph/nx_file_graph.py b/reme4/components/file_graph/nx_file_graph.py
new file mode 100644
index 00000000..c082ad04
--- /dev/null
+++ b/reme4/components/file_graph/nx_file_graph.py
@@ -0,0 +1,122 @@
+"""Networkx file-graph backend."""
+
+import pickle
+from pathlib import Path
+
+try:
+ import networkx as nx
+except ImportError:
+ nx = None
+
+from .base_file_graph import BaseFileGraph
+from ..component_registry import R
+from ...schema import FileLink, FileNode
+
+
+@R.register("nx")
+class NxFileGraph(BaseFileGraph):
+ """Networkx-backed file graph; uses FileLink.target_path for adjacency."""
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ if nx is None:
+ raise ImportError("NxFileGraph requires networkx — pip install networkx")
+ self._graph: nx.MultiDiGraph = nx.MultiDiGraph()
+ self._graph_file: Path = self.graph_path / f"{self.graph_name}_{self.graph_version}.pkl"
+
+ # -- Lifecycle ---------------------------------------------------------
+
+ async def _start(self) -> None:
+ await super()._start()
+ await self.load()
+
+ async def _close(self) -> None:
+ await self.dump()
+ await super()._close()
+
+ async def load(self) -> None:
+ """Load graph from pickle file; keep current graph on failure."""
+ if not self._graph_file.exists():
+ return
+ try:
+ with open(self._graph_file, "rb") as f:
+ self._graph = pickle.load(f)
+ n_real = sum(1 for _, d in self._graph.nodes(data=True) if "node" in d)
+ self.logger.info(f"Loaded {n_real} nodes from {self._graph_file}")
+ except Exception as e:
+ self.logger.exception(f"Failed to load {self._graph_file}: {e}")
+
+ async def dump(self) -> None:
+ """Persist graph to pickle via atomic rename."""
+ try:
+ tmp = self._graph_file.with_suffix(".tmp")
+ with open(tmp, "wb") as f:
+ pickle.dump(self._graph, f, protocol=pickle.HIGHEST_PROTOCOL)
+ tmp.replace(self._graph_file)
+ n_real = sum(1 for _, d in self._graph.nodes(data=True) if "node" in d)
+ self.logger.info(f"Saved {n_real} nodes to {self._graph_file}")
+ except Exception as e:
+ self.logger.exception(f"Failed to write {self._graph_file}: {e}")
+
+ # -- Node CRUD ---------------------------------------------------------
+
+ async def upsert_nodes(self, nodes: list[FileNode]) -> None:
+ for node in nodes:
+ path = node.path
+ if self._graph.has_node(path):
+ # Drop outgoing edges; inbound stay intact.
+ self._graph.remove_edges_from(list(self._graph.out_edges(path, keys=True)))
+ self._graph.add_node(path, node=node) # promotes virtual node if present
+ # Missing targets become attr-less virtual nodes.
+ self._graph.add_edges_from((path, lnk.target_path, {"link": lnk}) for lnk in node.links if lnk.target_path)
+
+ async def delete_nodes(self, paths: list[str]) -> None:
+ for path in paths:
+ if not self._graph.has_node(path):
+ continue
+ self._graph.remove_edges_from(list(self._graph.out_edges(path, keys=True)))
+ # Demote to virtual: keep inbound edges, drop node payload.
+ self._graph.nodes[path].pop("node", None)
+ if self._graph.in_degree(path) == 0:
+ self._graph.remove_node(path) # remove orphan virtual node
+
+ async def get_nodes(self, paths: list[str] | None = None) -> list[FileNode]:
+ nodes_view = self._graph.nodes
+ if paths is None:
+ return [d["node"] for _, d in nodes_view(data=True) if "node" in d]
+ return [nodes_view[path]["node"] for path in paths if path in nodes_view and "node" in nodes_view[path]]
+
+ async def rebuild_links(self) -> None:
+ """Rebuild all edges from real node payloads; drop virtual nodes."""
+ self._graph.remove_edges_from(list(self._graph.edges(keys=True)))
+ virtual = [n for n, d in self._graph.nodes(data=True) if "node" not in d]
+ self._graph.remove_nodes_from(virtual)
+ self._graph.add_edges_from(
+ (path, lnk.target_path, {"link": lnk})
+ for path, data in self._graph.nodes(data=True)
+ for lnk in data["node"].links
+ if lnk.target_path
+ )
+
+ async def clear(self):
+ """Remove all nodes and edges, and remove persisted file."""
+ self._graph.clear()
+ self._graph_file.unlink(missing_ok=True)
+
+ # -- Link access -------------------------------------------------------
+
+ async def get_outlinks(self, path: str) -> list[FileLink]:
+ nodes_view = self._graph.nodes
+ if path not in nodes_view or "node" not in nodes_view[path]:
+ return []
+ return [
+ d["link"]
+ for _, target, d in self._graph.out_edges(path, data=True)
+ if "link" in d and "node" in nodes_view[target]
+ ]
+
+ async def get_inlinks(self, path: str) -> list[FileLink]:
+ nodes_view = self._graph.nodes
+ if path not in nodes_view or "node" not in nodes_view[path]:
+ return []
+ return [d["link"] for _, _, d in self._graph.in_edges(path, data=True) if "link" in d]
diff --git a/reme4/components/file_parser/__init__.py b/reme4/components/file_parser/__init__.py
new file mode 100644
index 00000000..3e20d0bf
--- /dev/null
+++ b/reme4/components/file_parser/__init__.py
@@ -0,0 +1,7 @@
+"""File parser components."""
+
+from .bare_file_parser import BareFileParser
+from .base_file_parser import BaseFileParser
+from .default_file_parser import DefaultFileParser
+
+__all__ = ["BareFileParser", "BaseFileParser", "DefaultFileParser"]
diff --git a/reme4/components/file_parser/bare_file_parser.py b/reme4/components/file_parser/bare_file_parser.py
new file mode 100644
index 00000000..e0ecef05
--- /dev/null
+++ b/reme4/components/file_parser/bare_file_parser.py
@@ -0,0 +1,22 @@
+"""Stat-only parser for attachment/binary files."""
+
+from pathlib import Path
+
+from .base_file_parser import BaseFileParser
+from ..component_registry import R
+from ...schema import FileChunk, FileNode
+
+
+@R.register("bare")
+class BareFileParser(BaseFileParser):
+ """Stat-only parser for attachment/binary files.
+
+ No content read, no chunking, no link extraction. The resulting FileNode
+ has empty links and chunk_ids; front_matter carries mime and size so
+ retrieval can filter by file type without reopening the file.
+ """
+
+ async def parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]:
+ file_path = Path(path)
+ stat = file_path.stat()
+ return FileNode(path=self._get_relative_path(path), st_mtime=stat.st_mtime, links=[], chunk_ids=[]), []
diff --git a/reme4/components/file_parser/base_file_parser.py b/reme4/components/file_parser/base_file_parser.py
new file mode 100644
index 00000000..30595385
--- /dev/null
+++ b/reme4/components/file_parser/base_file_parser.py
@@ -0,0 +1,30 @@
+"""Abstract base for file parsers."""
+
+from abc import abstractmethod
+from pathlib import Path
+
+from ..base_component import BaseComponent
+from ...enumeration import ComponentEnum
+from ...schema import FileChunk, FileNode
+
+
+class BaseFileParser(BaseComponent):
+ """Abstract base for file parsers. Subclasses implement `parse`."""
+
+ component_type = ComponentEnum.FILE_PARSER
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ self.working_dir = self.app_context.app_config.working_dir if self.app_context else ""
+
+ def _get_relative_path(self, path: str | Path) -> str:
+ """Return path relative to working_dir, or absolute path if outside."""
+ file_path = Path(path).absolute()
+ try:
+ return str(file_path.relative_to(Path(self.working_dir).absolute()))
+ except ValueError:
+ return str(file_path)
+
+ @abstractmethod
+ async def parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]:
+ """Parse a file into (node, chunks)."""
diff --git a/reme4/components/file_parser/default_file_parser.py b/reme4/components/file_parser/default_file_parser.py
new file mode 100644
index 00000000..69a7c694
--- /dev/null
+++ b/reme4/components/file_parser/default_file_parser.py
@@ -0,0 +1,123 @@
+"""Default file parser with byte-based overlapping chunking."""
+
+import re
+from bisect import bisect_right
+from pathlib import Path
+
+import aiofiles
+import yaml
+
+from .base_file_parser import BaseFileParser
+from ..component_registry import R
+from ...schema import FileChunk, FileFrontMatter, FileLink, FileNode
+
+# Single-pass wikilink + optional Dataview predicate.
+# Covers: [[X]] / [[X#h]] / [[X|alias]] / pred:: [[X]] / [pred:: [[X]]]
+# - predicate group: optional leading '[' (Dataview inline-bracket form), an identifier,
+# then '::' — the whole prefix is non-capturing-optional so bare wikilinks still match.
+# - target / anchor: target stops before '#', '|', '[', ']'; anchor stops before '|', '[', ']'.
+# - alias '|...': consumed but not captured (we don't need display text).
+_LINK_RE = re.compile(
+ r"(?:\[?\s*(?P[A-Za-z][\w-]*)\s*::\s*)?"
+ r"\[\[\s*(?P[^\[\]|#]+?)"
+ r"(?:#(?P[^\[\]|]+?))?"
+ r"\s*(?:\|[^\[\]]*?)?\s*\]\]",
+)
+
+
+@R.register("default")
+class DefaultFileParser(BaseFileParser):
+ """Parser that splits files into byte-based overlapping chunks."""
+
+ def __init__(self, encoding: str = "utf-8", chunk_byte_size: int = 10000, overlap_byte_size: int = 100, **kwargs):
+ super().__init__(**kwargs)
+ self.encoding = encoding
+ self.chunk_byte_size = max(100, chunk_byte_size)
+ self.overlap_byte_size = max(4, overlap_byte_size)
+
+ @staticmethod
+ def parse_links(content: str, source_path: str) -> list[FileLink]:
+ """Extract wikilinks with optional Dataview predicate as outgoing FileLinks."""
+ links: list[FileLink] = []
+ for m in _LINK_RE.finditer(content):
+ target = m["target"].strip()
+ if not target:
+ continue
+ anchor = m["anchor"]
+ links.append(
+ FileLink(
+ source_path=source_path,
+ target_path=target,
+ target_anchor=anchor.strip() if anchor else None,
+ predicate=m["predicate"],
+ ),
+ )
+ return links
+
+ @staticmethod
+ def _parse_front_matter(text: str) -> tuple[FileFrontMatter, str]:
+ """Parse YAML front matter delimited by ---, return (front_matter, remaining)."""
+ if not text.startswith("---"):
+ return FileFrontMatter(), text
+ end_idx = text.find("\n---", 3)
+ if end_idx == -1:
+ return FileFrontMatter(), text
+ try:
+ data = yaml.safe_load(text[3:end_idx].strip()) or {}
+ front_matter = FileFrontMatter(**(data if isinstance(data, dict) else {}))
+ except yaml.YAMLError:
+ front_matter = FileFrontMatter()
+ return front_matter, text[end_idx + 4 :].lstrip("\n")
+
+ async def parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]:
+ file_path = Path(path)
+ stat = file_path.stat()
+ rel_path = self._get_relative_path(path)
+
+ async with aiofiles.open(file_path, encoding=self.encoding) as f:
+ text = await f.read()
+
+ if not text:
+ return FileNode(path=rel_path, st_mtime=stat.st_mtime), []
+
+ front_matter, content = self._parse_front_matter(text)
+ if not content:
+ return FileNode(path=rel_path, st_mtime=stat.st_mtime, front_matter=front_matter), []
+
+ links = self.parse_links(content, rel_path)
+ chunks = self._chunk_content(content, rel_path)
+ chunk_ids = [c.id for c in chunks]
+ return (
+ FileNode(
+ path=rel_path,
+ st_mtime=stat.st_mtime,
+ front_matter=front_matter,
+ links=links,
+ chunk_ids=chunk_ids,
+ ),
+ chunks,
+ )
+
+ def _chunk_content(self, content: str, rel_path: str) -> list[FileChunk]:
+ """Split content into overlapping byte-range chunks with line numbers."""
+ content_bytes = content.encode(self.encoding)
+ newline_positions = [i for i, b in enumerate(content_bytes) if b == ord("\n")]
+ chunks: list[FileChunk] = []
+ step = self.chunk_byte_size - self.overlap_byte_size
+ start = 0
+
+ while start < len(content_bytes):
+ end = min(start + self.chunk_byte_size, len(content_bytes))
+ chunk_text = content_bytes[start:end].decode(self.encoding, errors="ignore")
+ start_line = bisect_right(newline_positions, start - 1) + 1
+ end_line = bisect_right(newline_positions, end - 1) + 1
+ if content_bytes[end - 1] == ord("\n"):
+ end_line -= 1
+ chunks.append(
+ FileChunk(path=rel_path, start_line=start_line, end_line=end_line, text=chunk_text).set_hash_id(),
+ )
+ if end >= len(content_bytes):
+ break
+ start += step
+
+ return chunks
diff --git a/reme4/components/file_store/__init__.py b/reme4/components/file_store/__init__.py
new file mode 100644
index 00000000..7739fb99
--- /dev/null
+++ b/reme4/components/file_store/__init__.py
@@ -0,0 +1,14 @@
+"""File store module.
+
+In-memory + JSONL backend for the (file → chunks) graph. Subclass
+`BaseFileStore` to add other backends; only `LocalFileStore` is
+shipped today.
+"""
+
+from .base_file_store import BaseFileStore
+from .local_file_store import LocalFileStore
+
+__all__ = [
+ "BaseFileStore",
+ "LocalFileStore",
+]
diff --git a/reme4/components/file_store/base_file_store.py b/reme4/components/file_store/base_file_store.py
new file mode 100644
index 00000000..5cc6edd9
--- /dev/null
+++ b/reme4/components/file_store/base_file_store.py
@@ -0,0 +1,100 @@
+"""Abstract base for file store backends."""
+
+from abc import abstractmethod
+
+from ..base_component import BaseComponent
+from ..embedding import BaseEmbeddingModel
+from ..file_graph import BaseFileGraph
+from ..keyword_index import BaseKeywordIndex
+from ...enumeration import ComponentEnum
+from ...schema import FileChunk, FileNode, FileLink
+
+
+class BaseFileStore(BaseComponent):
+ """Abstract base for file store backends."""
+
+ component_type = ComponentEnum.FILE_STORE
+
+ def __init__(
+ self,
+ store_name: str,
+ embedding_model: str = "default",
+ keyword_index: str = "default",
+ file_graph: str = "default",
+ store_version: str = "v1",
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ from ..embedding import OpenAIEmbeddingModel
+ from ..file_graph import LocalFileGraph
+ from ..keyword_index import BM25Index
+
+ self.store_name = store_name or self.name
+ self.store_version = store_version
+ if not embedding_model and not keyword_index:
+ raise ValueError("At least one of embedding_model or keyword_index must be set.")
+
+ self.embedding_model = self.bind(embedding_model, BaseEmbeddingModel, default_factory=OpenAIEmbeddingModel)
+ self.keyword_index = self.bind(keyword_index, BaseKeywordIndex, default_factory=BM25Index)
+ self.file_graph = self.bind(file_graph, BaseFileGraph, default_factory=LocalFileGraph)
+ self.store_path = self.working_metadata_path / self.component_type.value / store_name
+ self.store_path.mkdir(parents=True, exist_ok=True)
+
+ async def _start(self) -> None:
+ """Probe embedding model; disable vector capability if it fails."""
+ if self.embedding_model is None:
+ return
+ if not await self.embedding_model.health_check():
+ self.logger.warning(f"{self.store_name}: embedding unhealthy, vector disabled")
+ self.embedding_model = None
+
+ def _disable_embedding(self, reason: str) -> None:
+ """Drop embedding after a runtime failure; keyword search still works."""
+ if self.embedding_model is None:
+ return
+ self.logger.error(f"{self.store_name}: embedding disabled, {reason}")
+ self.embedding_model = None
+
+ async def upsert_file(
+ self,
+ file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]],
+ ) -> None:
+ """Upsert a file and its chunks into the store."""
+
+ async def delete_by_path(self, path: str | list[str]) -> None:
+ """Delete files by their paths from the store."""
+
+ async def clear(self):
+ """Clear the store of all files and chunks."""
+
+ @abstractmethod
+ async def vector_search(self, query: str, limit: int, search_filter: dict) -> list[FileChunk]:
+ """Perform vector similarity search."""
+
+ @abstractmethod
+ async def keyword_search(self, query: str, limit: int, search_filter: dict) -> list[FileChunk]:
+ """Perform full-text keyword search."""
+
+ async def rebuild_links(self) -> None:
+ """Rebuild all edges from each node's link payload."""
+ if not self.file_graph:
+ raise RuntimeError("file_graph is required for delete_by_path")
+ return await self.file_graph.rebuild_links()
+
+ async def get_nodes(self, paths: list[str]) -> list[FileNode]:
+ """Return file nodes for the given paths (missing paths are skipped)."""
+ if not self.file_graph:
+ raise RuntimeError("file_graph is required for get_nodes")
+ return await self.file_graph.get_nodes(paths)
+
+ async def get_outlinks(self, path: str) -> list[FileLink]:
+ """Return outgoing links for *path*."""
+ if not self.file_graph:
+ raise RuntimeError("file_graph is required for delete_by_path")
+ return await self.file_graph.get_outlinks(path)
+
+ async def get_inlinks(self, path: str) -> list[FileLink]:
+ """Return incoming links for *path*."""
+ if not self.file_graph:
+ raise RuntimeError("file_graph is required for delete_by_path")
+ return await self.file_graph.get_inlinks(path)
diff --git a/reme4/components/file_store/local_file_store.py b/reme4/components/file_store/local_file_store.py
new file mode 100644
index 00000000..60b920d0
--- /dev/null
+++ b/reme4/components/file_store/local_file_store.py
@@ -0,0 +1,177 @@
+"""In-memory file store with JSONL persistence on close."""
+
+import aiofiles
+import numpy as np
+
+from .base_file_store import BaseFileStore
+from ..component_registry import R
+from ...schema import FileChunk, FileNode
+from ...utils import batch_cosine_similarity
+
+
+@R.register("local")
+class LocalFileStore(BaseFileStore):
+ """In-memory file store with deferred JSONL persistence."""
+
+ def __init__(self, encoding: str = "utf-8", **kwargs):
+ super().__init__(**kwargs)
+ self.encoding = encoding
+ self.file_chunks: dict[str, FileChunk] = {}
+ self.chunks_path = self.store_path / f"file_chunks_{self.store_version}.jsonl"
+
+ # Lifecycle
+
+ async def _start(self) -> None:
+ await super()._start()
+ await self.load()
+
+ async def _close(self) -> None:
+ await self.dump()
+ self.file_chunks.clear()
+ await super()._close()
+
+ async def load(self) -> None:
+ """Load chunks from JSONL file into memory."""
+ if not self.chunks_path.exists():
+ return
+ try:
+ async with aiofiles.open(self.chunks_path, encoding=self.encoding) as f:
+ async for line in f:
+ line = line.strip()
+ if line:
+ chunk = FileChunk.model_validate_json(line)
+ self.file_chunks[chunk.id] = chunk
+ self.logger.info(f"Loaded {len(self.file_chunks)} chunks from {self.chunks_path}")
+ except Exception as e:
+ self.logger.exception(f"Failed to load {self.chunks_path}: {e}")
+
+ async def dump(self) -> None:
+ """Persist chunks to JSONL via atomic rename, then cascade to keyword_index and file_graph."""
+ try:
+ tmp = self.chunks_path.with_suffix(".tmp")
+ async with aiofiles.open(tmp, "w", encoding=self.encoding) as f:
+ await f.write("\n".join(c.model_dump_json() for c in self.file_chunks.values()))
+ tmp.replace(self.chunks_path)
+ self.logger.info(f"Saved {len(self.file_chunks)} chunks to {self.chunks_path}")
+ except Exception as e:
+ self.logger.exception(f"Failed to write {self.chunks_path}: {e}")
+ if self.keyword_index:
+ await self.keyword_index.dump()
+ if self.file_graph:
+ await self.file_graph.dump()
+
+ # Base class interface
+
+ async def upsert_file(
+ self,
+ file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]],
+ ) -> None:
+ if not self.file_graph:
+ raise RuntimeError("file_graph is required for upsert_file")
+ if isinstance(file, tuple):
+ file = [file]
+
+ old_map = {n.path: n for n in await self.file_graph.get_nodes([node.path for node, _ in file])}
+
+ new_nodes: list[FileNode] = []
+ needs_embed: list[FileChunk] = []
+ keyword_docs: dict[str, str] = {}
+ for node, chunks in file:
+ old_node: FileNode | None = old_map.get(node.path)
+ cached = {}
+ if old_node and self.embedding_model:
+ for cid in old_node.chunk_ids:
+ old = self.file_chunks.pop(cid, None)
+ if old and old.embedding is not None:
+ cached[cid] = old.embedding
+
+ node.chunk_ids = []
+ for c in chunks:
+ if self.embedding_model and c.embedding is None:
+ if c.id in cached:
+ c.embedding = cached[c.id]
+ elif c.text:
+ needs_embed.append(c)
+ node.chunk_ids.append(c.id)
+ self.file_chunks[c.id] = c
+ if c.text:
+ keyword_docs[c.id] = c.text
+ new_nodes.append(node)
+
+ await self.file_graph.upsert_nodes(new_nodes)
+ if needs_embed and self.embedding_model:
+ try:
+ await self.embedding_model.get_node_embeddings(needs_embed)
+ except Exception as e:
+ self._disable_embedding(f"upsert: {type(e).__name__}: {e}")
+ if self.keyword_index and keyword_docs:
+ await self.keyword_index.add_docs(keyword_docs)
+
+ async def delete_by_path(self, path: str | list[str]) -> None:
+ if not self.file_graph:
+ raise RuntimeError("file_graph is required for delete_by_path")
+ if isinstance(path, str):
+ path = [path]
+ nodes = await self.file_graph.get_nodes(path)
+ if not nodes:
+ return
+ deleted_chunk_ids = [cid for n in nodes for cid in n.chunk_ids]
+ for cid in deleted_chunk_ids:
+ self.file_chunks.pop(cid, None)
+ await self.file_graph.delete_nodes([n.path for n in nodes])
+ if self.keyword_index and deleted_chunk_ids:
+ await self.keyword_index.delete_docs(deleted_chunk_ids)
+
+ async def clear(self) -> None:
+ if not self.file_graph:
+ raise RuntimeError("file_graph is required for clear")
+ self.file_chunks.clear()
+ self.chunks_path.unlink(missing_ok=True)
+ if self.keyword_index:
+ await self.keyword_index.clear()
+ await self.file_graph.clear()
+
+ # Search
+
+ async def vector_search(self, query: str, limit: int, search_filter: dict) -> list[FileChunk]:
+ if self.embedding_model is None or not query:
+ return []
+
+ try:
+ query_embedding = await self.embedding_model.get_embedding(query)
+ except Exception as e:
+ self._disable_embedding(f"search: {type(e).__name__}: {e}")
+ return []
+ if query_embedding is None:
+ return []
+
+ candidates = [c for c in self.file_chunks.values() if c.embedding is not None]
+ if not candidates:
+ return []
+
+ candidate_embeddings = np.stack([c.embedding for c in candidates])
+ similarities = batch_cosine_similarity(query_embedding.reshape(1, -1), candidate_embeddings)[0]
+
+ results = [
+ c.model_copy(update={"scores": {"vector": float(s), "score": float(s)}})
+ for c, s in zip(candidates, similarities)
+ ]
+ results.sort(key=lambda r: r.score, reverse=True)
+ return results[:limit]
+
+ async def keyword_search(self, query: str, limit: int, search_filter: dict) -> list[FileChunk]:
+ if not self.keyword_index:
+ return []
+
+ query = query.strip()
+ if not query:
+ return []
+
+ doc_id_score_dict = await self.keyword_index.retrieve(query, limit=limit)
+ results = []
+ for doc_id, score in doc_id_score_dict.items():
+ chunk = self.file_chunks.get(doc_id)
+ if chunk:
+ results.append(chunk.model_copy(update={"scores": {"keyword": score, "score": score}}))
+
+ return results
diff --git a/reme4/components/file_watcher/__init__.py b/reme4/components/file_watcher/__init__.py
new file mode 100644
index 00000000..0b11d995
--- /dev/null
+++ b/reme4/components/file_watcher/__init__.py
@@ -0,0 +1,9 @@
+"""File watcher implementations for monitoring file system changes."""
+
+from .base_file_watcher import BaseFileWatcher
+from .lite_file_watcher import LiteFileWatcher
+
+__all__ = [
+ "BaseFileWatcher",
+ "LiteFileWatcher",
+]
diff --git a/reme4/components/file_watcher/base_file_watcher.py b/reme4/components/file_watcher/base_file_watcher.py
new file mode 100644
index 00000000..3c6fbc9e
--- /dev/null
+++ b/reme4/components/file_watcher/base_file_watcher.py
@@ -0,0 +1,118 @@
+"""Abstract base for file watchers."""
+
+import asyncio
+from abc import abstractmethod
+from pathlib import Path
+
+from watchfiles import Change
+
+from ..base_component import BaseComponent
+from ..file_parser import BaseFileParser
+from ..file_store import BaseFileStore
+from ...enumeration import ComponentEnum
+
+
+class BaseFileWatcher(BaseComponent):
+ """Abstract base for file watchers. Subclasses implement watch_loop and event handlers."""
+
+ component_type = ComponentEnum.FILE_WATCHER
+
+ def __init__(
+ self,
+ watch_paths: list[str] | str,
+ suffix_filters: list[str] | None = None,
+ recursive: bool = True,
+ force_polling: bool = True,
+ debounce: int = 2000,
+ poll_delay_ms: int = 2000,
+ file_store: str = "default",
+ file_parser: str = "default",
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ from ..file_parser import DefaultFileParser
+ from ..file_store import LocalFileStore
+
+ watch_paths = [watch_paths] if isinstance(watch_paths, str) else watch_paths
+ base = self.working_path
+ self.watch_paths: list[Path] = [base / x for x in watch_paths if (base / x).exists()]
+ self.suffix_filters: list[str] = suffix_filters or ["md"]
+ self.recursive: bool = recursive
+ self.force_polling: bool = force_polling
+ self.debounce: int = debounce
+ self.poll_delay_ms: int = poll_delay_ms
+ self.file_store = self.bind(file_store, BaseFileStore, default_factory=LocalFileStore)
+ self.file_parser = self.bind(file_parser, BaseFileParser, default_factory=DefaultFileParser)
+ self._stop_event: asyncio.Event = asyncio.Event()
+ self._background_task: asyncio.Task | None = None
+ self._retry_interval: float = 10
+
+ async def _start(self):
+ self._stop_event = asyncio.Event()
+ self._background_task = asyncio.create_task(self._background_run())
+ self.logger.info(f"Started watching: {[str(p) for p in self.watch_paths]}")
+
+ async def _background_run(self):
+ """Sync store then enter watch loop."""
+ await self.update_store()
+ await self.watch_loop()
+
+ async def _close(self):
+ self._stop_event.set()
+ if self._background_task:
+ await self._background_task
+ self.logger.info("Stopped watching")
+
+ def watch_filter(self, _change: Change, path: str) -> bool:
+ """Return True if the file suffix matches the filter list."""
+ if not self.suffix_filters:
+ return True
+ return any(path.endswith("." + s.strip(".")) for s in self.suffix_filters)
+
+ def _get_relative_path(self, path: str | Path) -> str:
+ """Return path relative to working_dir, or absolute path if outside."""
+ file_path = Path(path).absolute()
+ try:
+ return str(file_path.relative_to(self.working_path.absolute()))
+ except ValueError:
+ return str(file_path)
+
+ def _get_absolute_path(self, path: str | Path) -> Path:
+ """Return absolute path; relative paths are resolved against working_dir."""
+ p = Path(path)
+ return p if p.is_absolute() else self.working_path / p
+
+ async def scan_existing_files(self) -> dict[str, Path]:
+ """Collect watchable files under watch_paths as {relative_path: absolute_path}."""
+ files: dict[str, Path] = {}
+ for path in self.watch_paths:
+ if not path.exists():
+ continue
+ candidates = [path] if path.is_file() else (path.rglob("*") if self.recursive else path.iterdir())
+ for p in candidates:
+ if p.is_file() and self.watch_filter(Change.added, str(p)):
+ files[self._get_relative_path(p)] = p.absolute()
+ return files
+
+ @abstractmethod
+ async def watch_loop(self):
+ """Watch for file changes and dispatch events."""
+
+ @abstractmethod
+ async def update_store(self, dump: bool = True) -> dict[str, int]:
+ """Sync the store with watch_paths; dump store if any changes and dump=True.
+
+ Returns counts {"added": int, "modified": int, "deleted": int}.
+ """
+
+ @abstractmethod
+ async def on_added(self, path: str | list[str]):
+ """Handle file added event (relative paths)."""
+
+ @abstractmethod
+ async def on_modified(self, path: str | list[str]):
+ """Handle file modified event (relative paths)."""
+
+ @abstractmethod
+ async def on_deleted(self, path: str | list[str]):
+ """Handle file deleted event (relative paths)."""
diff --git a/reme4/components/file_watcher/lite_file_watcher.py b/reme4/components/file_watcher/lite_file_watcher.py
new file mode 100644
index 00000000..da48b2b3
--- /dev/null
+++ b/reme4/components/file_watcher/lite_file_watcher.py
@@ -0,0 +1,129 @@
+"""Polling-based file watcher using watchfiles."""
+
+import asyncio
+
+from watchfiles import Change, awatch
+
+from .base_file_watcher import BaseFileWatcher
+from ..component_registry import R
+from ...schema import FileChunk, FileNode
+
+
+@R.register("lite")
+class LiteFileWatcher(BaseFileWatcher):
+ """Polling-based file watcher using watchfiles awatch."""
+
+ async def _interruptible_sleep(self):
+ """Sleep until stop or timeout, whichever comes first."""
+ try:
+ await asyncio.wait_for(self._stop_event.wait(), timeout=self._retry_interval)
+ except asyncio.TimeoutError:
+ pass
+
+ async def watch_loop(self):
+ if not self.watch_paths:
+ self.logger.warning("No watch paths specified")
+ return
+
+ while not self._stop_event.is_set():
+ valid_paths = [p for p in self.watch_paths if p.exists()]
+ if not valid_paths:
+ self.logger.warning(f"No valid paths, retrying in {self._retry_interval}s...")
+ await self._interruptible_sleep()
+ continue
+
+ invalid = set(self.watch_paths) - set(valid_paths)
+ if invalid:
+ self.logger.warning(f"Skipping invalid paths: {[str(p) for p in invalid]}")
+
+ try:
+ self.logger.info(f"Watching: {[str(p) for p in valid_paths]}")
+ async for changes in awatch(
+ *valid_paths,
+ watch_filter=self.watch_filter,
+ recursive=self.recursive,
+ force_polling=self.force_polling,
+ debounce=self.debounce,
+ poll_delay_ms=self.poll_delay_ms,
+ stop_event=self._stop_event,
+ ):
+ if self._stop_event.is_set():
+ break
+ await self._dispatch_changes(changes)
+ except Exception:
+ self.logger.exception(f"Watch error, retrying in {self._retry_interval}s...")
+ if not self._stop_event.is_set():
+ await self._interruptible_sleep()
+
+ async def _dispatch_changes(self, changes: set[tuple[Change, str]]):
+ """Classify raw changes and dispatch to event handlers."""
+ buckets: dict[Change, list[str]] = {Change.added: [], Change.modified: [], Change.deleted: []}
+ for c, p in changes:
+ if c in buckets:
+ buckets[c].append(self._get_relative_path(p))
+ for change, handler, label in (
+ (Change.added, self.on_added, "added"),
+ (Change.modified, self.on_modified, "modified"),
+ (Change.deleted, self.on_deleted, "deleted"),
+ ):
+ if buckets[change]:
+ self.logger.info(f"Detected {len(buckets[change])} {label} file(s)")
+ await handler(buckets[change])
+
+ async def update_store(self, dump: bool = True) -> dict[str, int]:
+ if self.file_store is None:
+ raise ValueError("file_store is not initialized!")
+
+ existing: dict[str, float] = {
+ rel: abs_p.stat().st_mtime for rel, abs_p in (await self.scan_existing_files()).items()
+ }
+ indexed: dict[str, float] = {n.path: n.st_mtime for n in await self.file_store.file_graph.get_nodes()}
+
+ to_delete = list(indexed.keys() - existing.keys())
+ to_add = list(existing.keys() - indexed.keys())
+ to_modify = [p for p in existing.keys() & indexed.keys() if existing[p] != indexed[p]]
+
+ if to_modify:
+ self.logger.info(f"Updating {len(to_modify)} modified file(s)")
+ await self.on_modified(to_modify)
+ if to_delete:
+ self.logger.info(f"Removing {len(to_delete)} deleted file(s)")
+ await self.on_deleted(to_delete)
+ if to_add:
+ self.logger.info(f"Indexing {len(to_add)} new file(s)")
+ await self.on_added(to_add)
+
+ changed = bool(to_add or to_modify or to_delete)
+ if not changed:
+ self.logger.info("Store is up to date")
+ if dump and changed:
+ await self.file_store.dump()
+ return {"added": len(to_add), "modified": len(to_modify), "deleted": len(to_delete)}
+
+ async def _parse_and_upsert(self, paths: list[str], action: str):
+ """Parse files and upsert into store. Shared by on_added / on_modified."""
+ if self.file_parser is None or self.file_store is None:
+ raise RuntimeError("file_parser or file_store is not initialized!")
+
+ parsed: list[tuple[FileNode, list[FileChunk]]] = []
+ for rel in paths:
+ abs_path = self._get_absolute_path(rel)
+ if abs_path.is_file():
+ self.logger.info(f"{action} file: {rel}")
+ parsed.append(await self.file_parser.parse(abs_path))
+ if parsed:
+ await self.file_store.delete_by_path([n.path for n, _ in parsed])
+ await self.file_store.upsert_file(parsed)
+
+ async def on_added(self, path: str | list[str]):
+ await self._parse_and_upsert([path] if isinstance(path, str) else path, "Adding")
+
+ async def on_modified(self, path: str | list[str]):
+ await self._parse_and_upsert([path] if isinstance(path, str) else path, "Updating")
+
+ async def on_deleted(self, path: str | list[str]):
+ if self.file_store is None:
+ raise RuntimeError("file_store is not initialized!")
+ paths = [path] if isinstance(path, str) else path
+ self.logger.info(f"Deleting {len(paths)} file(s)")
+ await self.file_store.delete_by_path(paths)
diff --git a/reme4/components/job/__init__.py b/reme4/components/job/__init__.py
new file mode 100644
index 00000000..0387572d
--- /dev/null
+++ b/reme4/components/job/__init__.py
@@ -0,0 +1,6 @@
+"""Job components for executing workflows."""
+
+from .base_job import BaseJob
+from .stream_job import StreamJob
+
+__all__ = ["BaseJob", "StreamJob"]
diff --git a/reme4/components/job/base_job.py b/reme4/components/job/base_job.py
new file mode 100644
index 00000000..e4681bf1
--- /dev/null
+++ b/reme4/components/job/base_job.py
@@ -0,0 +1,53 @@
+"""Base job component for sequential step execution."""
+
+from ..base_component import BaseComponent
+from ..component_registry import R
+from ..runtime_context import RuntimeContext
+from ...enumeration import ComponentEnum
+from ...schema import ComponentConfig, Response
+
+
+@R.register("base")
+class BaseJob(BaseComponent):
+ """Job that executes steps sequentially and returns a Response."""
+
+ component_type = ComponentEnum.JOB
+
+ def __init__(self, description: str, parameters: dict, steps: list[ComponentConfig | dict], **kwargs):
+ super().__init__(**kwargs)
+ self.description = description
+ self.parameters = parameters or {}
+ self.step_configs = steps or []
+
+ from ...steps import BaseStep
+
+ self.step_components: list[BaseStep] = []
+
+ async def _start(self) -> None:
+ """Resolve step configs into instantiated step components."""
+ assert self.app_context is not None, "app_context must be provided"
+ for raw in self.step_configs:
+ config = raw if isinstance(raw, ComponentConfig) else ComponentConfig(**raw)
+ if not config.backend:
+ raise ValueError("Step is missing the required 'backend' field")
+ step_cls = R.get(ComponentEnum.STEP, config.backend)
+ if not step_cls:
+ raise ValueError(f"Unregistered backend '{config.backend}' of type '{ComponentEnum.STEP}'")
+ params = config.model_dump()
+ params["app_context"] = self.app_context
+ self.step_components.append(step_cls(**params))
+
+ async def _close(self) -> None:
+ """Release all step components."""
+ self.step_components.clear()
+
+ async def __call__(self, **kwargs) -> Response:
+ """Execute all steps in order and return the final response."""
+ context = RuntimeContext(**kwargs)
+ try:
+ for step in self.step_components:
+ await step(context)
+ except Exception as e:
+ self.logger.exception(f"Failed to execute job: {e}")
+ context.response.answer = str(e)
+ return context.response
diff --git a/reme4/components/job/stream_job.py b/reme4/components/job/stream_job.py
new file mode 100644
index 00000000..89f1fab1
--- /dev/null
+++ b/reme4/components/job/stream_job.py
@@ -0,0 +1,21 @@
+"""Streaming job for real-time output delivery."""
+
+from .base_job import BaseJob
+from ..component_registry import R
+from ..runtime_context import RuntimeContext
+from ...enumeration import ChunkEnum
+
+
+@R.register("stream")
+class StreamJob(BaseJob):
+ """Job that streams chunks to a queue instead of returning a Response."""
+
+ async def __call__(self, **kwargs) -> None:
+ """Execute steps and stream output; errors are sent as ERROR chunks."""
+ context = RuntimeContext(**kwargs)
+ try:
+ for step in self.step_components:
+ await step(context)
+ except Exception as e:
+ await context.add_stream_string(str(e), ChunkEnum.ERROR)
+ await context.add_stream_done()
diff --git a/reme4/components/keyword_index/__init__.py b/reme4/components/keyword_index/__init__.py
new file mode 100644
index 00000000..f9969c6a
--- /dev/null
+++ b/reme4/components/keyword_index/__init__.py
@@ -0,0 +1,6 @@
+"""Keyword index components."""
+
+from .base_keyword_index import BaseKeywordIndex
+from .bm25_index import BM25Index
+
+__all__ = ["BaseKeywordIndex", "BM25Index"]
diff --git a/reme4/components/keyword_index/base_keyword_index.py b/reme4/components/keyword_index/base_keyword_index.py
new file mode 100644
index 00000000..0be21c58
--- /dev/null
+++ b/reme4/components/keyword_index/base_keyword_index.py
@@ -0,0 +1,70 @@
+"""Abstract base class for keyword index implementations."""
+
+from abc import abstractmethod
+from pathlib import Path
+
+from ..base_component import BaseComponent
+from ..tokenizer import BaseTokenizer
+from ...enumeration import ComponentEnum
+
+
+class BaseKeywordIndex(BaseComponent):
+ """Abstract base class for keyword index implementations."""
+
+ component_type = ComponentEnum.KEYWORD_INDEX
+
+ def __init__(self, tokenizer: str = "default", index_version: str = "v1", **kwargs):
+ super().__init__(**kwargs)
+ from ..tokenizer import RegexTokenizer
+
+ self.tokenizer = self.bind(tokenizer, BaseTokenizer, default_factory=RegexTokenizer)
+ self.index_version = index_version
+ self.index_path = self.working_metadata_path / self.component_type.value
+ self.index_path.mkdir(parents=True, exist_ok=True)
+
+ async def _start(self) -> None:
+ """Load existing index from disk if available."""
+ await self.load()
+
+ async def _close(self) -> None:
+ """Save index to disk on shutdown."""
+ await self.dump()
+
+ @property
+ def index_file(self) -> Path:
+ """Return the pickle file path derived from tokenizer name."""
+ if self.tokenizer is None:
+ raise RuntimeError("Tokenizer not initialized. Call start() first.")
+ name = type(self.tokenizer).__name__.replace("Tokenizer", "").lower()
+ return self.index_path / f"bm25_{name}_{self.index_version}.pkl"
+
+ def _tokenize(self, text: str) -> list[str]:
+ """Tokenize a text string into tokens."""
+ if self.tokenizer is None:
+ raise RuntimeError("Tokenizer not initialized. Call start() first.")
+ return self.tokenizer.tokenize([text])[0]
+
+ @abstractmethod
+ async def add_docs(self, docs_dict: dict[str, str]) -> None:
+ """Index or update documents. Mapping of doc_id to content."""
+
+ @abstractmethod
+ async def delete_docs(self, doc_ids: list[str]) -> None:
+ """Remove documents by their IDs."""
+
+ @abstractmethod
+ async def retrieve(self, query: str, limit: int = 3) -> dict[str, float]:
+ """Search documents. Returns {doc_id: score} sorted descending."""
+
+ @abstractmethod
+ async def clear(self) -> None:
+ """Reset index to empty state."""
+
+ async def reset_index(self, docs_dict: dict[str, str]) -> None:
+ """Clear index, re-add all documents, and persist."""
+ await self.clear()
+ await self.add_docs(docs_dict)
+ await self.dump()
+
+ async def optimize_index(self) -> None:
+ """Optimize index for performance. Override in subclass if needed."""
diff --git a/reme4/components/keyword_index/bm25_index.py b/reme4/components/keyword_index/bm25_index.py
new file mode 100644
index 00000000..81899828
--- /dev/null
+++ b/reme4/components/keyword_index/bm25_index.py
@@ -0,0 +1,206 @@
+"""BM25 search engine with persistent index support.
+
+Implements Okapi BM25 ranking with an inverted index for efficient
+document lookup, incremental updates, and pickle-based persistence.
+"""
+
+import math
+import pickle
+from collections import Counter
+from typing import TypedDict
+
+from .base_keyword_index import BaseKeywordIndex
+from ..component_registry import R
+
+
+class DocMeta(TypedDict):
+ """Per-document metadata: token count and unique token ID set."""
+
+ len: int
+ token_ids: set[int]
+
+
+@R.register("bm25")
+class BM25Index(BaseKeywordIndex):
+ """BM25 search engine with file-based persistence.
+
+ Args:
+ k1: Term frequency saturation parameter (default 1.5).
+ b: Document length normalization parameter (default 0.75).
+ """
+
+ def __init__(self, k1: float = 1.5, b: float = 0.75, **kwargs):
+ super().__init__(**kwargs)
+ self.k1 = k1
+ self.b = b
+ self.vocab: dict[str, int] = {} # token -> token_id
+ self.inverted_index: dict[int, dict[str, int]] = {} # token_id -> {doc_id: tf}
+ self.doc_meta: dict[str, DocMeta] = {} # doc_id -> metadata
+ self.total_len: int = 0
+ self._idf_cache: dict[int, float] = {}
+
+ # -- Properties -----------------------------------------------------------
+
+ @property
+ def n_docs(self) -> int:
+ """Number of indexed documents."""
+ return len(self.doc_meta)
+
+ @property
+ def avg_len(self) -> float:
+ """Average document length in tokens."""
+ return self.total_len / self.n_docs if self.n_docs > 0 else 0.0
+
+ # -- Internal helpers -----------------------------------------------------
+
+ def _tokens_to_ids(self, tokens: list[str]) -> list[int]:
+ """Map tokens to integer IDs, assigning new IDs on first encounter."""
+ ids = []
+ for token in tokens:
+ token = token.strip()
+ if token:
+ ids.append(self.vocab.setdefault(token, len(self.vocab)))
+ return ids
+
+ def _remove_doc(self, doc_id: str) -> None:
+ """Remove a single document from all internal structures."""
+ if doc_id not in self.doc_meta:
+ return
+ meta = self.doc_meta[doc_id]
+ self.total_len -= meta["len"]
+ for tid in meta["token_ids"]:
+ if tid in self.inverted_index:
+ self.inverted_index[tid].pop(doc_id, None)
+ if not self.inverted_index[tid]:
+ del self.inverted_index[tid]
+ del self.doc_meta[doc_id]
+
+ def _get_idf(self, token_id: int) -> float:
+ """Compute and cache IDF for a token ID."""
+ if token_id in self._idf_cache:
+ return self._idf_cache[token_id]
+ df = len(self.inverted_index.get(token_id, {}))
+ self._idf_cache[token_id] = math.log(1 + (self.n_docs - df + 0.5) / (df + 0.5)) if df else 0.0
+ return self._idf_cache[token_id]
+
+ # -- Public API -----------------------------------------------------------
+
+ async def add_docs(self, docs_dict: dict[str, str]) -> None:
+ """Index or update multiple documents. Mapping of doc_id to content."""
+ for doc_id, content in docs_dict.items():
+ if doc_id in self.doc_meta:
+ self._remove_doc(doc_id)
+ tokens = self._tokenize(content)
+ if not tokens:
+ continue
+ token_ids = self._tokens_to_ids(tokens)
+ token_counts = Counter(token_ids)
+ for tid, tf in token_counts.items():
+ self.inverted_index.setdefault(tid, {})[doc_id] = tf
+ self.doc_meta[doc_id] = {"len": len(token_ids), "token_ids": set(token_counts)}
+ self.total_len += len(token_ids)
+ self._idf_cache = {}
+
+ async def delete_docs(self, doc_ids: list[str]) -> None:
+ """Remove documents by their IDs."""
+ for doc_id in doc_ids:
+ self._remove_doc(doc_id)
+ self._idf_cache = {}
+
+ async def retrieve(self, query: str, limit: int = 3) -> dict[str, float]:
+ """Search documents. Returns {doc_id: score} sorted descending."""
+ query_ids = [self.vocab[t] for t in self._tokenize(query) if t in self.vocab]
+ if not query_ids or self.n_docs == 0:
+ return {}
+
+ scores: dict[str, float] = {}
+ avg_len = self.avg_len
+ for tid in query_ids:
+ if tid not in self.inverted_index:
+ continue
+ idf = self._get_idf(tid)
+ for doc_id, tf in self.inverted_index[tid].items():
+ doc_len = self.doc_meta[doc_id]["len"]
+ tf_score = tf * (self.k1 + 1) / (tf + self.k1 * (1 - self.b + self.b * doc_len / avg_len))
+ scores[doc_id] = scores.get(doc_id, 0.0) + idf * tf_score
+
+ return dict(sorted(scores.items(), key=lambda x: x[1], reverse=True)[:limit]) if scores else {}
+
+ async def dump(self) -> None:
+ """Persist index to disk via pickle (atomic rename)."""
+ try:
+ tmp = self.index_file.with_suffix(".tmp")
+ with open(tmp, "wb") as f:
+ pickle.dump(
+ {
+ "vocab": self.vocab,
+ "inverted_index": self.inverted_index,
+ "doc_meta": self.doc_meta,
+ "total_len": self.total_len,
+ "k1": self.k1,
+ "b": self.b,
+ },
+ f,
+ )
+ tmp.replace(self.index_file)
+ self.logger.info(f"Saved {self.n_docs} docs to {self.index_file}")
+ except Exception as e:
+ self.logger.exception(f"Failed to write {self.index_file}: {e}")
+
+ async def load(self) -> None:
+ """Load index from disk. No-op if file missing; clears index on corruption."""
+ if not self.index_file.exists():
+ return
+ try:
+ with open(self.index_file, "rb") as f:
+ data = pickle.load(f)
+ self.vocab = data["vocab"]
+ self.inverted_index = data["inverted_index"]
+ self.doc_meta = data["doc_meta"]
+ self.total_len = data.get("total_len", 0)
+ self.k1 = data.get("k1", 1.5)
+ self.b = data.get("b", 0.75)
+ self._idf_cache = {}
+ self.logger.info(f"Loaded {self.n_docs} docs from {self.index_file}")
+ except Exception as e:
+ self.logger.exception(f"Failed to load index: {e}")
+ self.index_file.unlink(missing_ok=True)
+ await self.clear()
+
+ async def clear(self) -> None:
+ """Reset index to empty state and remove persisted file."""
+ self.vocab = {}
+ self.inverted_index = {}
+ self.doc_meta = {}
+ self.total_len = 0
+ self._idf_cache = {}
+ self.index_file.unlink(missing_ok=True)
+
+ async def optimize_index(self) -> None:
+ """Rebuild vocab to remove unused tokens and compact token IDs."""
+ used_token_ids: set[int] = set()
+ for tid in self.inverted_index:
+ used_token_ids.add(tid)
+ if not used_token_ids:
+ await self.clear()
+ return
+
+ # Build compact ID mapping
+ old_to_new: dict[int, int] = {}
+ new_vocab: dict[str, int] = {}
+ for token, old_tid in self.vocab.items():
+ if old_tid in used_token_ids:
+ new_tid = len(new_vocab)
+ new_vocab[token] = new_tid
+ old_to_new[old_tid] = new_tid
+
+ # Rebuild inverted index and doc_meta with new IDs
+ new_inverted_index: dict[int, dict[str, int]] = {}
+ for old_tid, postings in self.inverted_index.items():
+ new_inverted_index[old_to_new[old_tid]] = postings
+ for meta in self.doc_meta.values():
+ meta["token_ids"] = {old_to_new[t] for t in meta["token_ids"] if t in old_to_new}
+
+ self.vocab = new_vocab
+ self.inverted_index = new_inverted_index
+ self._idf_cache = {}
diff --git a/reme4/components/prompt_handler.py b/reme4/components/prompt_handler.py
new file mode 100644
index 00000000..a9ae203a
--- /dev/null
+++ b/reme4/components/prompt_handler.py
@@ -0,0 +1,125 @@
+"""Prompt template loader and formatter with conditional-line and i18n support."""
+
+import inspect
+import json
+import re
+from pathlib import Path
+from string import Formatter
+
+import yaml
+
+# Matches a leading flag tag like "[verbose] some text".
+_FLAG_PATTERN = re.compile(r"^\[(\w+)]")
+
+
+class PromptHandler:
+ """Loads prompts from YAML/JSON or class-adjacent files and formats them.
+
+ Templates may carry a language suffix (``key_en``, ``key_zh``); ``get_prompt``
+ falls back to the bare key when no localized variant exists. ``prompt_format``
+ additionally supports per-line flags such as ``[verbose] extra text`` that
+ are kept only when the matching flag kwarg is truthy.
+ """
+
+ _SUPPORTED_EXTENSIONS = {".yaml", ".yml", ".json"}
+
+ def __init__(self, language: str = "", **kwargs):
+ # Only string entries are treated as prompts; other kwargs are ignored.
+ self.data: dict[str, str] = {k: v for k, v in kwargs.items() if isinstance(v, str)}
+ self.language: str = language.strip()
+
+ def load_prompt_by_file(
+ self,
+ prompt_file_path: str | Path | None = None,
+ overwrite: bool = True,
+ ) -> "PromptHandler":
+ """Load prompts from a YAML or JSON file; silently skip on any error."""
+ if prompt_file_path is None:
+ return self
+
+ path = Path(prompt_file_path)
+ if not path.exists() or path.suffix.lower() not in self._SUPPORTED_EXTENSIONS:
+ return self
+
+ try:
+ with path.open(encoding="utf-8") as f:
+ prompt_dict = yaml.safe_load(f) if path.suffix.lower() in (".yaml", ".yml") else json.load(f)
+ except (json.JSONDecodeError, yaml.YAMLError, OSError):
+ return self
+
+ return self.load_prompt_dict(prompt_dict, overwrite)
+
+ def load_prompt_by_class(self, cls: type, overwrite: bool = True) -> "PromptHandler":
+ """Load prompts from ``.yaml`` (or ``.yml``) next to `cls`."""
+ try:
+ base_path = Path(inspect.getfile(cls)).with_suffix("")
+ except (TypeError, OSError):
+ return self
+
+ for ext in (".yaml", ".yml"):
+ if (prompt_path := base_path.with_suffix(ext)).exists():
+ return self.load_prompt_by_file(prompt_path, overwrite)
+
+ return self
+
+ def load_prompt_dict(self, prompt_dict: dict | None = None, overwrite: bool = True) -> "PromptHandler":
+ """Merge string entries from `prompt_dict` into the in-memory store."""
+ if not isinstance(prompt_dict, dict):
+ return self
+
+ for key, value in prompt_dict.items():
+ if isinstance(value, str) and (overwrite or key not in self.data):
+ self.data[key] = value
+
+ return self
+
+ def get_prompt(self, prompt_name: str) -> str:
+ """Return the template, preferring the language-suffixed variant when set."""
+ for key in (f"{prompt_name}_{self.language}", prompt_name) if self.language else (prompt_name,):
+ if key in self.data:
+ return self.data[key].strip()
+
+ raise KeyError(f"Prompt '{prompt_name}' not found. Available: {list(self.data.keys())[:10]}")
+
+ def has_prompt(self, prompt_name: str) -> bool:
+ """True if either the localized or bare prompt is registered."""
+ keys = (f"{prompt_name}_{self.language}", prompt_name) if self.language else (prompt_name,)
+ return any(k in self.data for k in keys)
+
+ def list_prompts(self, language_filter: str | None = None) -> list[str]:
+ """List all keys, optionally filtered to those ending with ``_``."""
+ if language_filter is None:
+ return list(self.data.keys())
+ suffix = f"_{language_filter.strip()}"
+ return [k for k in self.data if k.endswith(suffix)]
+
+ def prompt_format(self, prompt_name: str, validate: bool = True, **kwargs) -> str:
+ """Render a prompt: strip inactive flag-lines, then ``str.format`` it.
+
+ Boolean kwargs are treated as flags controlling ``[flag]`` line filtering.
+ Remaining kwargs become positional substitutions for ``{var}`` placeholders.
+ With `validate=True`, missing substitutions raise ``ValueError``.
+ """
+ prompt = self.get_prompt(prompt_name)
+ flags = {k: v for k, v in kwargs.items() if isinstance(v, bool)}
+ formats = {k: v for k, v in kwargs.items() if not isinstance(v, bool)}
+
+ # Keep lines without flags; otherwise keep when at least one flag is enabled.
+ if flags:
+ lines = []
+ for line in prompt.split("\n"):
+ active_flags = _FLAG_PATTERN.findall(line)
+ cleaned = _FLAG_PATTERN.sub("", line).lstrip()
+ if not active_flags or any(flags.get(f, False) for f in active_flags):
+ lines.append(cleaned)
+ prompt = "\n".join(lines)
+
+ if validate:
+ required = {f for _, f, _, _ in Formatter().parse(prompt) if f is not None}
+ if missing := required - set(formats.keys()):
+ raise ValueError(f"Missing format variables for '{prompt_name}': {sorted(missing)}")
+
+ return prompt.format(**formats).strip() if formats else prompt
+
+ def __repr__(self) -> str:
+ return f"PromptHandler(language='{self.language}', num_prompts={len(self.data)})"
diff --git a/reme4/components/runtime_context.py b/reme4/components/runtime_context.py
new file mode 100644
index 00000000..76ebaf16
--- /dev/null
+++ b/reme4/components/runtime_context.py
@@ -0,0 +1,87 @@
+"""Per-request runtime context shared across steps and jobs."""
+
+import asyncio
+
+from ..enumeration import ChunkEnum
+from ..schema import Response, StreamChunk
+
+
+class RuntimeContext:
+ """Scratch space for a single execution.
+
+ Holds the response object, an optional stream queue, and a free-form
+ data dict accessed via mapping-style operators.
+ """
+
+ def __init__(
+ self,
+ response: Response | None = None,
+ stream_queue: asyncio.Queue | None = None,
+ **kwargs,
+ ):
+ self.response: Response = response or Response()
+ self.stream_queue: asyncio.Queue | None = stream_queue
+ self.data: dict = kwargs
+
+ def get(self, key: str, default=None):
+ """Get a value from the data dict."""
+ return self.data.get(key, default)
+
+ def update(self, data: dict) -> "RuntimeContext":
+ """Merge data into the context."""
+ self.data.update(data)
+ return self
+
+ def __getitem__(self, key: str):
+ return self.data[key]
+
+ def __setitem__(self, key: str, value):
+ self.data[key] = value
+
+ def __delitem__(self, key: str):
+ del self.data[key]
+
+ def __contains__(self, key: str) -> bool:
+ return key in self.data
+
+ @property
+ def stream(self) -> bool:
+ """Whether streaming is enabled."""
+ return self.stream_queue is not None
+
+ @classmethod
+ def from_context(cls, context: "RuntimeContext | None" = None, **kwargs) -> "RuntimeContext":
+ """Reuse or create a RuntimeContext."""
+ # Reuse the existing context (merging kwargs) or create a new one.
+ if context is None:
+ return cls(**kwargs)
+ context.update(kwargs)
+ return context
+
+ async def _enqueue(self, chunk: StreamChunk) -> None:
+ """Put a chunk on the stream queue."""
+ if self.stream_queue is None:
+ raise RuntimeError("Stream queue not initialized")
+ await self.stream_queue.put(chunk)
+
+ async def add_stream_string(self, chunk: str, chunk_type: ChunkEnum) -> "RuntimeContext":
+ """Emit a text chunk to the stream queue."""
+ # Emit a text chunk to the stream queue.
+ await self._enqueue(StreamChunk(chunk_type=chunk_type, chunk=chunk))
+ return self
+
+ async def add_stream_done(self) -> "RuntimeContext":
+ """Emit the terminal DONE marker to close the stream."""
+ # Emit the terminal DONE marker to close the stream.
+ await self._enqueue(StreamChunk(chunk_type=ChunkEnum.DONE, chunk="", done=True))
+ return self
+
+ def apply_mapping(self, mapping: dict[str, str]) -> "RuntimeContext":
+ """Copy data[source] into data[target] for each mapping pair."""
+ # Copy data[source] into data[target] for each {source: target} pair.
+ if not mapping:
+ return self
+ for source, target in mapping.items():
+ if source in self.data:
+ self.data[target] = self.data[source]
+ return self
diff --git a/reme4/components/service/__init__.py b/reme4/components/service/__init__.py
new file mode 100644
index 00000000..41d8f7b0
--- /dev/null
+++ b/reme4/components/service/__init__.py
@@ -0,0 +1,11 @@
+"""Service components for exposing jobs via different protocols."""
+
+from .base_service import BaseService
+from .http_service import HttpService
+from .mcp_service import MCPService
+
+__all__ = [
+ "BaseService",
+ "HttpService",
+ "MCPService",
+]
diff --git a/reme4/components/service/base_service.py b/reme4/components/service/base_service.py
new file mode 100644
index 00000000..8fceeadb
--- /dev/null
+++ b/reme4/components/service/base_service.py
@@ -0,0 +1,48 @@
+"""Base service class for exposing jobs via HTTP, MCP, etc."""
+
+from abc import abstractmethod
+from typing import TYPE_CHECKING
+
+from ..base_component import BaseComponent
+from ..job.base_job import BaseJob
+from ...enumeration import ComponentEnum
+
+if TYPE_CHECKING:
+ from ...application import Application
+
+
+class BaseService(BaseComponent):
+ """Base class for services that expose jobs via HTTP, MCP, etc."""
+
+ component_type = ComponentEnum.SERVICE
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ self.service = None
+
+ @abstractmethod
+ def build_service(self, app: "Application") -> None:
+ """Initialize the underlying service framework."""
+
+ @abstractmethod
+ def add_job(self, job: BaseJob) -> None:
+ """Register a single job with the service."""
+
+ @abstractmethod
+ def start_service(self, app: "Application") -> None:
+ """Start serving requests."""
+
+ def add_jobs(self, app: "Application") -> None:
+ """Register all jobs from the application context."""
+ for name, job in app.context.jobs.items():
+ try:
+ self.add_job(job)
+ self.logger.info(f"Added job: {name}")
+ except Exception as e:
+ self.logger.error(f"Failed to add job {name}: {e}")
+
+ def run_app(self, app: "Application") -> None:
+ """Build, populate, and start the service."""
+ self.build_service(app)
+ self.add_jobs(app)
+ self.start_service(app)
diff --git a/reme4/components/service/http_service.py b/reme4/components/service/http_service.py
new file mode 100644
index 00000000..a1092947
--- /dev/null
+++ b/reme4/components/service/http_service.py
@@ -0,0 +1,99 @@
+"""HTTP service implementation for ReMe."""
+
+import asyncio
+import json
+import os
+import warnings
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
+from typing import TYPE_CHECKING
+
+import uvicorn
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import StreamingResponse
+
+from .base_service import BaseService
+from ..component_registry import R
+from ..job import BaseJob, StreamJob
+from ...constants import REME_DEFAULT_HOST, REME_DEFAULT_PORT, REME_SERVICE_INFO
+from ...schema import Request, Response
+from ...utils import execute_stream_task
+
+if TYPE_CHECKING:
+ from ...application import Application
+
+
+@R.register("http")
+class HttpService(BaseService):
+ """HTTP service: normal jobs -> JSON endpoints, stream jobs -> SSE endpoints."""
+
+ def __init__(self, host: str = REME_DEFAULT_HOST, port: int = REME_DEFAULT_PORT, **kwargs):
+ super().__init__(**kwargs)
+ self.host: str = host
+ self.port: int = port
+
+ def _add_job(self, job: BaseJob) -> None:
+ async def execute_endpoint(request: Request) -> Response:
+ return await job(**request.model_dump(exclude_none=True))
+
+ self.service.post(path=f"/{job.name}", response_model=Response, description=job.description)(execute_endpoint)
+
+ def _add_stream_job(self, job: StreamJob) -> None:
+ async def execute_stream_endpoint(request: Request) -> StreamingResponse:
+ stream_queue = asyncio.Queue()
+ task = asyncio.create_task(job(stream_queue=stream_queue, **request.model_dump(exclude_none=True)))
+
+ async def generate_stream() -> AsyncGenerator[bytes, None]:
+ async for chunk in execute_stream_task(
+ stream_queue=stream_queue,
+ task=task,
+ task_name=job.name,
+ output_format="bytes",
+ ):
+ assert isinstance(chunk, bytes)
+ yield chunk
+
+ return StreamingResponse(generate_stream(), media_type="text/event-stream")
+
+ self.service.post(f"/{job.name}")(execute_stream_endpoint)
+
+ def add_job(self, job: BaseJob) -> None:
+ if isinstance(job, StreamJob):
+ self._add_stream_job(job)
+ else:
+ self._add_job(job)
+
+ def build_service(self, app: "Application") -> None:
+ @asynccontextmanager
+ async def lifespan(_: FastAPI):
+ await app.start()
+ service_info = json.dumps({"host": self.host, "port": self.port})
+ os.environ[REME_SERVICE_INFO] = service_info
+ self.logger.info(f"ReMe Service started: {REME_SERVICE_INFO}={service_info}")
+ yield
+ await app.close()
+
+ self.service = FastAPI(title=app.config.app_name, lifespan=lifespan)
+ self.service.add_middleware(
+ CORSMiddleware, # type: ignore[arg-type]
+ allow_origins=["*"],
+ allow_credentials=True,
+ allow_methods=["*"],
+ allow_headers=["*"],
+ )
+
+ def start_service(self, app: "Application") -> None:
+ # uvicorn 0.41 still imports websockets.legacy / WebSocketServerProtocol
+ # on startup; silence those specific lines since we don't use WebSocket.
+ warnings.filterwarnings(
+ "ignore",
+ category=DeprecationWarning,
+ message=r".*websockets\.legacy is deprecated.*",
+ )
+ warnings.filterwarnings(
+ "ignore",
+ category=DeprecationWarning,
+ message=r".*WebSocketServerProtocol is deprecated.*",
+ )
+ uvicorn.run(self.service, host=self.host, port=self.port, **self.kwargs)
diff --git a/reme4/components/service/mcp_service.py b/reme4/components/service/mcp_service.py
new file mode 100644
index 00000000..8f22450d
--- /dev/null
+++ b/reme4/components/service/mcp_service.py
@@ -0,0 +1,71 @@
+"""MCP (Model Context Protocol) service implementation."""
+
+import json
+import os
+from contextlib import asynccontextmanager
+from typing import TYPE_CHECKING
+
+from fastmcp import FastMCP
+from fastmcp.server.server import Transport
+from fastmcp.tools import FunctionTool
+
+from .base_service import BaseService
+from ..component_registry import R
+from ..job import StreamJob, BaseJob
+from ...constants import REME_DEFAULT_HOST, REME_DEFAULT_PORT, REME_SERVICE_INFO
+
+if TYPE_CHECKING:
+ from ...application import Application
+
+
+@R.register("mcp")
+class MCPService(BaseService):
+ """Expose jobs as MCP (Model Context Protocol) tools."""
+
+ def __init__(
+ self,
+ transport: Transport = "sse",
+ host: str = REME_DEFAULT_HOST,
+ port: int = REME_DEFAULT_PORT,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ self.transport: Transport = transport
+ self.host: str = host
+ self.port: int = port
+
+ def build_service(self, app: "Application") -> None:
+ @asynccontextmanager
+ async def lifespan(_: FastMCP):
+ await app.start()
+ service_info = json.dumps({"host": self.host, "port": self.port})
+ os.environ[REME_SERVICE_INFO] = service_info
+ self.logger.info(f"ReMe MCP Service started: {REME_SERVICE_INFO}={service_info}")
+ yield
+ await app.close()
+
+ self.service = FastMCP(name=app.config.app_name, lifespan=lifespan)
+
+ def add_job(self, job: "BaseJob") -> None:
+ if isinstance(job, StreamJob):
+ return
+
+ async def execute_tool(**kwargs):
+ response = await job(**kwargs)
+ return response.answer
+
+ self.service.add_tool(
+ FunctionTool(
+ name=job.name,
+ description=job.description,
+ fn=execute_tool,
+ parameters=job.parameters or None,
+ ),
+ )
+
+ def start_service(self, app: "Application") -> None:
+ transport_kwargs = {}
+ if self.transport != "stdio":
+ transport_kwargs["host"] = self.host
+ transport_kwargs["port"] = self.port
+ self.service.run(transport=self.transport, show_banner=False, **transport_kwargs)
diff --git a/reme4/components/tokenizer/__init__.py b/reme4/components/tokenizer/__init__.py
new file mode 100644
index 00000000..ff364470
--- /dev/null
+++ b/reme4/components/tokenizer/__init__.py
@@ -0,0 +1,11 @@
+"""Tokenizer component module."""
+
+from .base_tokenizer import BaseTokenizer
+from .jieba_tokenizer import JiebaTokenizer
+from .regex_tokenizer import RegexTokenizer
+
+__all__ = [
+ "BaseTokenizer",
+ "JiebaTokenizer",
+ "RegexTokenizer",
+]
diff --git a/reme4/components/tokenizer/base_tokenizer.py b/reme4/components/tokenizer/base_tokenizer.py
new file mode 100644
index 00000000..713c2ecb
--- /dev/null
+++ b/reme4/components/tokenizer/base_tokenizer.py
@@ -0,0 +1,44 @@
+"""Abstract base class for tokenizers."""
+
+from abc import abstractmethod
+from pathlib import Path
+
+import aiofiles
+
+from ..base_component import BaseComponent
+from ...enumeration import ComponentEnum
+
+
+class BaseTokenizer(BaseComponent):
+ """Base tokenizer. Subclasses must implement `tokenize`. Loads stopwords on start."""
+
+ component_type = ComponentEnum.TOKENIZER
+ DEFAULT_STOPWORDS_PATH = Path(__file__).parent / "stopwords"
+
+ def __init__(self, stopwords_path: str | Path | None = None, **kwargs):
+ super().__init__(**kwargs)
+ self.stopwords_path = Path(stopwords_path) if stopwords_path else self.DEFAULT_STOPWORDS_PATH
+ self._stopwords: set[str] = set()
+
+ async def _start(self) -> None:
+ """Load stopwords from file."""
+ if not self.stopwords_path.exists():
+ self.logger.warning(f"Stopwords file not found: {self.stopwords_path}")
+ return
+ async with aiofiles.open(self.stopwords_path, encoding="utf-8") as f:
+ content = await f.read()
+ self._stopwords = {line.strip().lower() for line in content.splitlines() if line.strip()}
+ self.logger.info(f"Loaded {len(self._stopwords)} stopwords from {self.stopwords_path}")
+
+ async def _close(self) -> None:
+ """Clear stopwords."""
+ self._stopwords.clear()
+
+ @property
+ def stopwords(self) -> set[str]:
+ """Get the loaded stopwords."""
+ return self._stopwords
+
+ @abstractmethod
+ def tokenize(self, texts: list[str], **kwargs) -> list[list[str]]:
+ """Tokenize a list of texts."""
diff --git a/reme4/components/tokenizer/jieba_tokenizer.py b/reme4/components/tokenizer/jieba_tokenizer.py
new file mode 100644
index 00000000..4391c89c
--- /dev/null
+++ b/reme4/components/tokenizer/jieba_tokenizer.py
@@ -0,0 +1,27 @@
+"""Jieba tokenizer for Chinese text segmentation."""
+
+from .base_tokenizer import BaseTokenizer
+from ..component_registry import R
+
+
+@R.register("jieba")
+class JiebaTokenizer(BaseTokenizer):
+ """Tokenizer using jieba for Chinese text segmentation."""
+
+ def __init__(self, filter_stopwords: bool = True, **kwargs):
+ super().__init__(**kwargs)
+ self.filter_stopwords = filter_stopwords
+
+ def tokenize(self, texts: list[str], lower: bool = True, **kwargs) -> list[list[str]]:
+ """Tokenize texts using jieba."""
+ import jieba
+
+ result = []
+ for text in texts:
+ tokens = jieba.cut(text)
+ if lower:
+ tokens = [x.lower() for x in tokens]
+ if self.filter_stopwords and self._stopwords:
+ tokens = [t for t in tokens if t not in self._stopwords]
+ result.append(tokens)
+ return result
diff --git a/reme4/components/tokenizer/regex_tokenizer.py b/reme4/components/tokenizer/regex_tokenizer.py
new file mode 100644
index 00000000..4379a60e
--- /dev/null
+++ b/reme4/components/tokenizer/regex_tokenizer.py
@@ -0,0 +1,31 @@
+"""Regex tokenizer with Chinese character splitting."""
+
+import re
+from .base_tokenizer import BaseTokenizer
+from ..component_registry import R
+
+
+@R.register("regex")
+class RegexTokenizer(BaseTokenizer):
+ """Tokenizer using regex: splits Chinese chars individually, extracts non-Chinese words."""
+
+ WORD_PATTERN = re.compile(r"(?u)\b\w\w+\b") # 2+ char words
+ CHINESE_PATTERN = re.compile(r"[一-鿿]") # single Chinese char
+
+ def __init__(self, filter_stopwords: bool = True, **kwargs):
+ super().__init__(**kwargs)
+ self.filter_stopwords = filter_stopwords
+
+ def tokenize(self, texts: list[str], lower: bool = True, **kwargs) -> list[list[str]]:
+ """Tokenize texts. Extracts Chinese chars, then non-Chinese words from remaining text."""
+ result = []
+ for text in texts:
+ # Extract Chinese chars individually, then non-Chinese words
+ tokens = self.CHINESE_PATTERN.findall(text)
+ tokens.extend(self.WORD_PATTERN.findall(self.CHINESE_PATTERN.sub(" ", text)))
+ if lower:
+ tokens = [t.lower() for t in tokens]
+ if self.filter_stopwords and self._stopwords:
+ tokens = [t for t in tokens if t not in self._stopwords]
+ result.append(tokens)
+ return result
diff --git a/reme4/components/tokenizer/stopwords b/reme4/components/tokenizer/stopwords
new file mode 100644
index 00000000..fc3c61d0
--- /dev/null
+++ b/reme4/components/tokenizer/stopwords
@@ -0,0 +1,1395 @@
+--
+?
+“
+”
+》
+－－
+able
+about
+above
+according
+accordingly
+across
+actually
+after
+afterwards
+again
+against
+ain't
+all
+allow
+allows
+almost
+alone
+along
+already
+also
+although
+always
+am
+among
+amongst
+an
+and
+another
+any
+anybody
+anyhow
+anyone
+anything
+anyway
+anyways
+anywhere
+apart
+appear
+appreciate
+appropriate
+are
+aren't
+around
+as
+a's
+aside
+ask
+asking
+associated
+at
+available
+away
+awfully
+be
+became
+because
+become
+becomes
+becoming
+been
+before
+beforehand
+behind
+being
+believe
+below
+beside
+besides
+best
+better
+between
+beyond
+both
+brief
+but
+by
+came
+can
+cannot
+cant
+can't
+cause
+causes
+certain
+certainly
+changes
+clearly
+c'mon
+co
+com
+come
+comes
+concerning
+consequently
+consider
+considering
+contain
+containing
+contains
+corresponding
+could
+couldn't
+course
+c's
+currently
+definitely
+described
+despite
+did
+didn't
+different
+do
+does
+doesn't
+doing
+done
+don't
+down
+downwards
+during
+each
+edu
+eg
+eight
+either
+else
+elsewhere
+enough
+entirely
+especially
+et
+etc
+even
+ever
+every
+everybody
+everyone
+everything
+everywhere
+ex
+exactly
+example
+except
+far
+few
+fifth
+first
+five
+followed
+following
+follows
+for
+former
+formerly
+forth
+four
+from
+further
+furthermore
+get
+gets
+getting
+given
+gives
+go
+goes
+going
+gone
+got
+gotten
+greetings
+had
+hadn't
+happens
+hardly
+has
+hasn't
+have
+haven't
+having
+he
+hello
+help
+hence
+her
+here
+hereafter
+hereby
+herein
+here's
+hereupon
+hers
+herself
+he's
+hi
+him
+himself
+his
+hither
+hopefully
+how
+howbeit
+however
+i'd
+ie
+if
+ignored
+i'll
+i'm
+immediate
+in
+inasmuch
+inc
+indeed
+indicate
+indicated
+indicates
+inner
+insofar
+instead
+into
+inward
+is
+isn't
+it
+it'd
+it'll
+its
+it's
+itself
+i've
+just
+keep
+keeps
+kept
+know
+known
+knows
+last
+lately
+later
+latter
+latterly
+least
+less
+lest
+let
+let's
+like
+liked
+likely
+little
+look
+looking
+looks
+ltd
+mainly
+many
+may
+maybe
+me
+mean
+meanwhile
+merely
+might
+more
+moreover
+most
+mostly
+much
+must
+my
+myself
+name
+namely
+nd
+near
+nearly
+necessary
+need
+needs
+neither
+never
+nevertheless
+new
+next
+nine
+no
+nobody
+non
+none
+noone
+nor
+normally
+not
+nothing
+novel
+now
+nowhere
+obviously
+of
+off
+often
+oh
+ok
+okay
+old
+on
+once
+one
+ones
+only
+onto
+or
+other
+others
+otherwise
+ought
+our
+ours
+ourselves
+out
+outside
+over
+overall
+own
+particular
+particularly
+per
+perhaps
+placed
+please
+plus
+possible
+presumably
+probably
+provides
+que
+quite
+qv
+rather
+rd
+re
+really
+reasonably
+regarding
+regardless
+regards
+relatively
+respectively
+right
+said
+same
+saw
+say
+saying
+says
+second
+secondly
+see
+seeing
+seem
+seemed
+seeming
+seems
+seen
+self
+selves
+sensible
+sent
+serious
+seriously
+seven
+several
+shall
+she
+should
+shouldn't
+since
+six
+so
+some
+somebody
+somehow
+someone
+something
+sometime
+sometimes
+somewhat
+somewhere
+soon
+sorry
+specified
+specify
+specifying
+still
+sub
+such
+sup
+sure
+take
+taken
+tell
+tends
+th
+than
+thank
+thanks
+thanx
+that
+thats
+that's
+the
+their
+theirs
+them
+themselves
+then
+thence
+there
+thereafter
+thereby
+therefore
+therein
+theres
+there's
+thereupon
+these
+they
+they'd
+they'll
+they're
+they've
+think
+third
+this
+thorough
+thoroughly
+those
+though
+three
+through
+throughout
+thru
+thus
+to
+together
+too
+took
+toward
+towards
+tried
+tries
+truly
+try
+trying
+t's
+twice
+two
+un
+under
+unfortunately
+unless
+unlikely
+until
+unto
+up
+upon
+us
+use
+used
+useful
+uses
+using
+usually
+value
+various
+very
+via
+viz
+vs
+want
+wants
+was
+wasn't
+way
+we
+we'd
+welcome
+well
+we'll
+went
+were
+we're
+weren't
+we've
+what
+whatever
+what's
+when
+whence
+whenever
+where
+whereafter
+whereas
+whereby
+wherein
+where's
+whereupon
+wherever
+whether
+which
+while
+whither
+who
+whoever
+whole
+whom
+who's
+whose
+why
+will
+willing
+wish
+with
+within
+without
+wonder
+won't
+would
+wouldn't
+yes
+yet
+you
+you'd
+you'll
+your
+you're
+yours
+yourself
+yourselves
+you've
+zero
+zt
+ZT
+zz
+ZZ
+一
+一下
+一些
+一切
+一则
+一天
+一定
+一方面
+一旦
+一时
+一来
+一样
+一次
+一片
+一直
+一致
+一般
+一起
+一边
+一面
+万一
+上下
+上升
+上去
+上来
+上述
+上面
+下列
+下去
+下来
+下面
+不一
+不久
+不仅
+不会
+不但
+不光
+不单
+不变
+不只
+不可
+不同
+不够
+不如
+不得
+不怕
+不惟
+不成
+不拘
+不敢
+不断
+不是
+不比
+不然
+不特
+不独
+不管
+不能
+不要
+不论
+不足
+不过
+不问
+与
+与其
+与否
+与此同时
+专门
+且
+两者
+严格
+严重
+个
+个人
+个别
+中小
+中间
+丰富
+临
+为
+为主
+为了
+为什么
+为什麽
+为何
+为着
+主张
+主要
+举行
+乃
+乃至
+么
+之
+之一
+之前
+之后
+之後
+之所以
+之类
+乌乎
+乎
+乘
+也
+也好
+也是
+也罢
+了
+了解
+争取
+于
+于是
+于是乎
+云云
+互相
+产生
+人们
+人家
+什么
+什么样
+什麽
+今后
+今天
+今年
+今後
+仍然
+从
+从事
+从而
+他
+他人
+他们
+他的
+代替
+以
+以上
+以下
+以为
+以便
+以免
+以前
+以及
+以后
+以外
+以後
+以来
+以至
+以至于
+以致
+们
+任
+任何
+任凭
+任务
+企图
+伟大
+似乎
+似的
+但
+但是
+何
+何况
+何处
+何时
+作为
+你
+你们
+你的
+使得
+使用
+例如
+依
+依照
+依靠
+促进
+保持
+俺
+俺们
+倘
+倘使
+倘或
+倘然
+倘若
+假使
+假如
+假若
+做到
+像
+允许
+充分
+先后
+先後
+先生
+全部
+全面
+兮
+共同
+关于
+其
+其一
+其中
+其二
+其他
+其余
+其它
+其实
+其次
+具体
+具体地说
+具体说来
+具有
+再者
+再说
+冒
+冲
+决定
+况且
+准备
+几
+几乎
+几时
+凭
+凭借
+出去
+出来
+出现
+分别
+则
+别
+别的
+别说
+到
+前后
+前者
+前进
+前面
+加之
+加以
+加入
+加强
+十分
+即
+即令
+即使
+即便
+即或
+即若
+却不
+原来
+又
+及
+及其
+及时
+及至
+双方
+反之
+反应
+反映
+反过来
+反过来说
+取得
+受到
+变成
+另
+另一方面
+另外
+只是
+只有
+只要
+只限
+叫
+叫做
+召开
+叮咚
+可
+可以
+可是
+可能
+可见
+各
+各个
+各人
+各位
+各地
+各种
+各级
+各自
+合理
+同
+同一
+同时
+同样
+后来
+后面
+向
+向着
+吓
+吗
+否则
+吧
+吧哒
+吱
+呀
+呃
+呕
+呗
+呜
+呜呼
+呢
+周围
+呵
+呸
+呼哧
+咋
+和
+咚
+咦
+咱
+咱们
+咳
+哇
+哈
+哈哈
+哉
+哎
+哎呀
+哎哟
+哗
+哟
+哦
+哩
+哪
+哪个
+哪些
+哪儿
+哪天
+哪年
+哪怕
+哪样
+哪边
+哪里
+哼
+哼唷
+唉
+啊
+啐
+啥
+啦
+啪达
+喂
+喏
+喔唷
+嗡嗡
+嗬
+嗯
+嗳
+嘎
+嘎登
+嘘
+嘛
+嘻
+嘿
+因
+因为
+因此
+因而
+固然
+在
+在下
+地
+坚决
+坚持
+基本
+处理
+复杂
+多
+多少
+多数
+多次
+大力
+大多数
+大大
+大家
+大批
+大约
+大量
+失去
+她
+她们
+她的
+好的
+好象
+如
+如上所述
+如下
+如何
+如其
+如果
+如此
+如若
+存在
+宁
+宁可
+宁愿
+宁肯
+它
+它们
+它们的
+它的
+安全
+完全
+完成
+实现
+实际
+宣布
+容易
+密切
+对
+对于
+对应
+将
+少数
+尔后
+尚且
+尤其
+就
+就是
+就是说
+尽
+尽管
+属于
+岂但
+左右
+巨大
+巩固
+己
+已经
+帮助
+常常
+并
+并不
+并不是
+并且
+并没有
+广大
+广泛
+应当
+应用
+应该
+开外
+开始
+开展
+引起
+强烈
+强调
+归
+当
+当前
+当时
+当然
+当着
+形成
+彻底
+彼
+彼此
+往
+往往
+待
+後来
+後面
+得
+得出
+得到
+心里
+必然
+必要
+必须
+怎
+怎么
+怎么办
+怎么样
+怎样
+怎麽
+总之
+总是
+总的来看
+总的来说
+总的说来
+总结
+总而言之
+恰恰相反
+您
+意思
+愿意
+慢说
+成为
+我
+我们
+我的
+或
+或是
+或者
+战斗
+所
+所以
+所有
+所谓
+打
+扩大
+把
+抑或
+拿
+按
+按照
+换句话说
+换言之
+据
+掌握
+接着
+接著
+故
+故此
+整个
+方便
+方面
+旁人
+无宁
+无法
+无论
+既
+既是
+既然
+时候
+明显
+明确
+是
+是否
+是的
+显然
+显著
+普通
+普遍
+更加
+曾经
+替
+最后
+最大
+最好
+最後
+最近
+最高
+有
+有些
+有关
+有利
+有力
+有所
+有效
+有时
+有点
+有的
+有着
+有著
+望
+朝
+朝着
+本
+本着
+来
+来着
+极了
+构成
+果然
+果真
+某
+某个
+某些
+根据
+根本
+欢迎
+正在
+正如
+正常
+此
+此外
+此时
+此间
+毋宁
+每
+每个
+每天
+每年
+每当
+比
+比如
+比方
+比较
+毫不
+没有
+沿
+沿着
+注意
+深入
+清楚
+满足
+漫说
+焉
+然则
+然后
+然後
+然而
+照
+照着
+特别是
+特殊
+特点
+现代
+现在
+甚么
+甚而
+甚至
+用
+由
+由于
+由此可见
+的
+的话
+目前
+直到
+直接
+相似
+相信
+相反
+相同
+相对
+相对而言
+相应
+相当
+相等
+省得
+看出
+看到
+看来
+看看
+看见
+真是
+真正
+着
+着呢
+矣
+知道
+确定
+离
+积极
+移动
+突出
+突然
+立即
+第
+等
+等等
+管
+紧接着
+纵
+纵令
+纵使
+纵然
+练习
+组成
+经
+经常
+经过
+结合
+结果
+给
+绝对
+继续
+继而
+维持
+综上所述
+罢了
+考虑
+者
+而
+而且
+而况
+而外
+而已
+而是
+而言
+联系
+能
+能否
+能够
+腾
+自
+自个儿
+自从
+自各儿
+自家
+自己
+自身
+至
+至于
+良好
+若
+若是
+若非
+范围
+莫若
+获得
+虽
+虽则
+虽然
+虽说
+行为
+行动
+表明
+表示
+被
+要
+要不
+要不是
+要不然
+要么
+要是
+要求
+规定
+觉得
+认为
+认真
+认识
+让
+许多
+论
+设使
+设若
+该
+说明
+诸位
+谁
+谁知
+赶
+起
+起来
+起见
+趁
+趁着
+越是
+跟
+转动
+转变
+转贴
+较
+较之
+边
+达到
+迅速
+过
+过去
+过来
+运用
+还是
+还有
+这
+这个
+这么
+这么些
+这么样
+这么点儿
+这些
+这会儿
+这儿
+这就是说
+这时
+这样
+这点
+这种
+这边
+这里
+这麽
+进入
+进步
+进而
+进行
+连
+连同
+适应
+适当
+适用
+逐步
+逐渐
+通常
+通过
+造成
+遇到
+遭到
+避免
+那
+那个
+那么
+那么些
+那么样
+那些
+那会儿
+那儿
+那时
+那样
+那边
+那里
+那麽
+部分
+鄙人
+采取
+里面
+重大
+重新
+重要
+鉴于
+问题
+防止
+阿
+附近
+限制
+除
+除了
+除此之外
+除非
+随
+随着
+随著
+集中
+需要
+非但
+非常
+非徒
+靠
+顺
+顺着
+首先
+高兴
+是不是
+说说
diff --git a/reme4/config/__init__.py b/reme4/config/__init__.py
new file mode 100644
index 00000000..c2903189
--- /dev/null
+++ b/reme4/config/__init__.py
@@ -0,0 +1,8 @@
+"""Config"""
+
+from .config_parser import parse_args, resolve_app_config
+
+__all__ = [
+ "parse_args",
+ "resolve_app_config",
+]
diff --git a/reme4/config/config_parser.py b/reme4/config/config_parser.py
new file mode 100644
index 00000000..92fce784
--- /dev/null
+++ b/reme4/config/config_parser.py
@@ -0,0 +1,219 @@
+"""Parser for YAML config with CLI argument overrides."""
+
+import json
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+# Config files are looked up relative to this module's directory
+_CONFIG_DIR = Path(__file__).parent
+# Extensions in priority order: yaml > yml > json when stems collide
+_SUPPORTED_EXTS = (".yaml", ".yml", ".json")
+_ENV_VAR_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?}")
+# Strings like "007" / "00501" must stay as strings, not be coerced to numbers
+_LEADING_ZERO_RE = re.compile(r"^-?0\d")
+
+
+def _repl(m: re.Match) -> str:
+ name: str = m.group(1)
+ # group(2) is None when the placeholder has no `:-default` part
+ default: str | None = m.group(2)
+ v = os.environ.get(name)
+ if v is None:
+ if default is not None:
+ return default
+ raise ValueError(f"Config references undefined env var: {name}")
+ return v
+
+
+def _expand_env_vars(value: Any) -> Any:
+ """Recursively expand `${VAR}` / `${VAR:-default}` placeholders in strings."""
+ if isinstance(value, str):
+ return _ENV_VAR_RE.sub(_repl, value)
+ if isinstance(value, dict):
+ return {k: _expand_env_vars(v) for k, v in value.items()}
+ if isinstance(value, list):
+ return [_expand_env_vars(v) for v in value]
+ return value
+
+
+def _discover_configs() -> dict[str, Path]:
+ """Pre-scan config directory: maps file stem (name without ext) -> Path."""
+ discovered: dict[str, Path] = {}
+ if _CONFIG_DIR.is_dir():
+ # Sort by ext priority so registration order is deterministic across filesystems
+ files = sorted(
+ (p for p in _CONFIG_DIR.iterdir() if p.is_file() and p.suffix in _SUPPORTED_EXTS),
+ key=lambda p: (_SUPPORTED_EXTS.index(p.suffix), p.name),
+ )
+ for p in files:
+ discovered.setdefault(p.stem, p)
+ return discovered
+
+
+_CONFIG_REGISTRY = _discover_configs()
+
+
+def parse_dot_notation(dot_list: list[str]) -> dict:
+ """Parse "key.subkey=value" strings into nested dict."""
+ result: dict = {}
+ for item in dot_list:
+ if "=" not in item:
+ raise ValueError(f"Invalid dot notation format (missing '='): {item}")
+ key_path, value_str = item.split("=", 1)
+ keys = key_path.split(".")
+ current = result
+ for key in keys[:-1]:
+ if key in current and not isinstance(current[key], dict):
+ raise ValueError(f"Cannot set nested key '{key_path}': '{key}' is already a value")
+ current = current.setdefault(key, {})
+ # Symmetric to the prefix check above: refuse scalar-over-dict overwrite
+ last_key = keys[-1]
+ if last_key in current and isinstance(current[last_key], dict):
+ raise ValueError(f"Cannot overwrite nested dict at '{key_path}' with scalar value")
+ current[last_key] = _convert_value(value_str)
+ return result
+
+
+def _convert_value(value_str: str) -> Any:
+ """Convert string to appropriate Python type.
+
+ Only converts "true"/"false" (case-insensitive) to boolean.
+ Use JSON format (e.g., '"yes"', '"no"') to preserve these as strings.
+ Leading-zero strings (e.g., "007", "00501") are kept as strings.
+ """
+ s = value_str.strip()
+ lower = s.lower()
+
+ # Handle special values (null, bool)
+ if lower in ("none", "null"):
+ return None
+ if lower == "true":
+ return True
+ if lower == "false":
+ return False
+
+ # Skip int/float for leading-zero strings to keep zip codes / ids intact
+ if not _LEADING_ZERO_RE.match(s):
+ for converter in (int, float):
+ try:
+ return converter(s)
+ except ValueError:
+ continue
+
+ # JSON handles lists, dicts, and explicitly-quoted strings
+ try:
+ return json.loads(s)
+ except (ValueError, json.JSONDecodeError):
+ pass
+
+ # Fallback to original string
+ return s
+
+
+def _load_config(name_or_path: str, encoding: str = "utf-8") -> dict:
+ """Load a YAML or JSON config file.
+
+ First check if name_or_path matches a pre-discovered config (key in _CONFIG_REGISTRY).
+ If not, treat as a file path and load directly.
+ """
+ # 1. Try pre-discovered configs first
+ if name_or_path in _CONFIG_REGISTRY:
+ return _read_config_file(_CONFIG_REGISTRY[name_or_path], encoding)
+
+ # 2. Treat as file path
+ p = Path(name_or_path)
+ if p.suffix in _SUPPORTED_EXTS:
+ if not p.exists():
+ raise FileNotFoundError(f"Config file not found: {p}")
+ return _read_config_file(p, encoding)
+
+ known = ", ".join(sorted(_CONFIG_REGISTRY)) if _CONFIG_REGISTRY else "none"
+ raise FileNotFoundError(f"Config file not found: {name_or_path}. Available: {known}")
+
+
+def _read_config_file(path: Path, encoding: str = "utf-8") -> dict:
+ """Read YAML or JSON file based on extension. Expands ${ENV_VAR}."""
+ with path.open(encoding=encoding) as f:
+ if path.suffix == ".json":
+ result = json.load(f)
+ else:
+ result = yaml.safe_load(f)
+ if result is None:
+ return {}
+ return _expand_env_vars(result)
+
+
+def _deep_merge(base: dict, update: dict) -> dict:
+ """Recursively merge dicts."""
+ result = base.copy()
+ for k, v in update.items():
+ if k in result and isinstance(result[k], dict) and isinstance(v, dict):
+ result[k] = _deep_merge(result[k], v)
+ else:
+ result[k] = v
+ return result
+
+
+def _strip_arg_dashes(arg: str) -> str:
+ """Strip a single leading `--` or `-` prefix (not all leading dashes)."""
+ if arg.startswith("--"):
+ return arg[2:]
+ if arg.startswith("-"):
+ return arg[1:]
+ return arg
+
+
+def parse_args(*args) -> tuple[str, dict]:
+ """Parse CLI args: first arg is action, rest are key=value pairs.
+
+ Usage: reme app config=paw.yaml service.name=test
+ Returns: (action, parsed_kv_dict)
+ """
+ if not args:
+ raise ValueError("No arguments provided")
+
+ first = _strip_arg_dashes(args[0])
+ if "=" in first:
+ raise ValueError(f"First argument must be action, got: {args[0]}")
+
+ kvs: list[str] = []
+ for raw in args[1:]:
+ arg = _strip_arg_dashes(raw)
+ if "=" in arg:
+ kvs.append(arg)
+
+ parsed = parse_dot_notation(kvs) if kvs else {}
+ return first, parsed
+
+
+def resolve_app_config(**kwargs) -> dict:
+ """Resolve full app-start config: load `config=path` file, fall back to
+ `default`, then deep-merge with the remaining kwargs as overrides.
+ """
+ from ..utils import get_logger
+
+ logger = get_logger()
+ configs: list[dict] = []
+
+ # `config=path` arrives as a string here; `config.foo=bar` arrives as a
+ # nested dict and is left in `kwargs` to be merged as a normal override.
+ config_value = kwargs.get("config")
+ if isinstance(config_value, str):
+ kwargs.pop("config")
+ logger.info(f"Loading config: {config_value}")
+ configs.append(_load_config(config_value))
+ elif "default" in _CONFIG_REGISTRY:
+ logger.info("No config specified, loading 'default'")
+ configs.append(_load_config("default"))
+
+ configs.append(kwargs)
+
+ merged: dict = {}
+ for cfg in configs:
+ merged = _deep_merge(merged, cfg)
+
+ return merged
diff --git a/reme4/config/default.yaml b/reme4/config/default.yaml
new file mode 100644
index 00000000..1a501912
--- /dev/null
+++ b/reme4/config/default.yaml
@@ -0,0 +1,169 @@
+service:
+ backend: http
+# backend: mcp
+
+jobs:
+ - backend: base
+ name: demo
+ description: "demo job description"
+ parameters:
+ type: object
+ properties:
+ query:
+ type: string
+ description: "query"
+ min_score:
+ type: number
+ description: "min score"
+ default: 0.5
+ required:
+ - query
+ steps:
+ - backend: demo_echo_step1
+ - backend: demo_echo_step2
+
+ - backend: base
+ name: version
+ description: "return reme4 package version"
+ parameters:
+ type: object
+ properties: {}
+ steps:
+ - backend: version_step
+
+ - backend: base
+ name: health_check
+ description: "return a concise health-check snapshot of reme4 components"
+ parameters:
+ type: object
+ properties: {}
+ steps:
+ - backend: health_check_step
+
+ - backend: base
+ name: help
+ description: "list all registered jobs with their metadata"
+ parameters:
+ type: object
+ properties: {}
+ steps:
+ - backend: help_step
+
+ - backend: base
+ name: reindex
+ description: "wipe the file store and rebuild it from the watcher's tracked files"
+ parameters:
+ type: object
+ properties: {}
+ steps:
+ - backend: reindex_step
+
+ - backend: base
+ name: search
+ description: "hybrid search over file_store: vector + keyword fused via RRF"
+ parameters:
+ type: object
+ properties:
+ query:
+ type: string
+ description: "search query"
+ limit:
+ type: integer
+ description: "max results to return"
+ default: 5
+ min_score:
+ type: number
+ description: "minimum fused score threshold (RRF scores are small; default 0 disables filter)"
+ default: 0.0
+ vector_weight:
+ type: number
+ description: "weight for vector results in [0, 1]; keyword weight = 1 - vector_weight"
+ default: 0.7
+ candidate_multiplier:
+ type: number
+ description: "candidate pool multiplier per branch (capped at 200)"
+ default: 3.0
+ expand_links:
+ type: boolean
+ description: "attach outlinks/inlinks (with neighbor meta) to each result"
+ default: true
+ max_links_per_direction:
+ type: integer
+ description: "max neighbors shown per direction per result"
+ default: 10
+ required:
+ - query
+ steps:
+ - backend: search_step
+
+ - backend: stream
+ name: stream_demo
+ description: "stream demo job: repeat query 10x and stream char-by-char"
+ parameters:
+ type: object
+ properties:
+ query:
+ type: string
+ description: "query to echo"
+ repeat:
+ type: integer
+ description: "number of times to repeat the query"
+ default: 10
+ interval:
+ type: number
+ description: "seconds between chunks"
+ default: 0.1
+ required:
+ - query
+ steps:
+ - backend: stream_demo_step1
+ - backend: stream_demo_step2
+
+components:
+ # 1. tokenizer — no dependencies
+ tokenizer:
+ default:
+ backend: regex
+
+ # 2. embedding_model — no dependencies
+ embedding_model:
+ default:
+ backend: openai
+ model_name: text-embedding-v4
+ dimensions: 1024
+
+ # 3. file_graph — no dependencies
+ file_graph:
+ default:
+ backend: local
+
+ # 4. file_parser — no dependencies
+ file_parser:
+ default:
+ backend: default
+
+ # 5. keyword_index — depends on tokenizer
+ keyword_index:
+ default:
+ backend: bm25
+ tokenizer: default
+
+ # 6. file_store — depends on embedding_model / keyword_index / file_graph
+ file_store:
+ default:
+ backend: local
+ store_name: default
+# embedding_model: default
+ embedding_model: ""
+ keyword_index: default
+ file_graph: default
+
+ # 7. file_watcher — depends on file_store / file_parser
+ file_watcher:
+ default:
+ backend: lite
+ watch_paths:
+ - MEMORY.md
+ - memory
+ file_store: default
+ file_parser: default
\ No newline at end of file
diff --git a/reme4/constants.py b/reme4/constants.py
new file mode 100644
index 00000000..fce66e64
--- /dev/null
+++ b/reme4/constants.py
@@ -0,0 +1,7 @@
+"""Constants"""
+
+REME_SERVICE_INFO = "REME_SERVICE_INFO"
+
+REME_DEFAULT_HOST = "127.0.0.1"
+
+REME_DEFAULT_PORT = 2333
diff --git a/reme4/enumeration/__init__.py b/reme4/enumeration/__init__.py
new file mode 100644
index 00000000..9887ddee
--- /dev/null
+++ b/reme4/enumeration/__init__.py
@@ -0,0 +1,9 @@
+"""Enumeration"""
+
+from .chunk_enum import ChunkEnum
+from .component_enum import ComponentEnum
+
+__all__ = [
+ "ChunkEnum",
+ "ComponentEnum",
+]
diff --git a/reme4/enumeration/chunk_enum.py b/reme4/enumeration/chunk_enum.py
new file mode 100644
index 00000000..d397be9e
--- /dev/null
+++ b/reme4/enumeration/chunk_enum.py
@@ -0,0 +1,21 @@
+"""Chunk enumeration module."""
+
+from enum import Enum
+
+
+class ChunkEnum(str, Enum):
+ """Enumeration of possible chunk categories for stream processing."""
+
+ THINK = "think"
+
+ CONTENT = "content"
+
+ TOOL_CALL = "tool_call"
+
+ TOOL_RESULT = "tool_result"
+
+ USAGE = "usage"
+
+ ERROR = "error"
+
+ DONE = "done"
diff --git a/reme4/enumeration/component_enum.py b/reme4/enumeration/component_enum.py
new file mode 100644
index 00000000..c3e30a89
--- /dev/null
+++ b/reme4/enumeration/component_enum.py
@@ -0,0 +1,37 @@
+"""Component enumeration module."""
+
+from enum import Enum
+
+
+class ComponentEnum(str, Enum):
+ """Enumeration of component types for dependency injection and registration."""
+
+ BASE = "base"
+
+ AS_LLM = "as_llm"
+
+ AS_LLM_FORMATTER = "as_llm_formatter"
+
+ AS_TOKEN_COUNTER = "as_token_counter"
+
+ EMBEDDING_MODEL = "embedding_model"
+
+ FILE_PARSER = "file_parser"
+
+ FILE_STORE = "file_store"
+
+ FILE_GRAPH = "file_graph"
+
+ FILE_WATCHER = "file_watcher"
+
+ KEYWORD_INDEX = "keyword_index"
+
+ SERVICE = "service"
+
+ CLIENT = "client"
+
+ STEP = "step"
+
+ JOB = "job"
+
+ TOKENIZER = "tokenizer"
diff --git a/reme4/reme.py b/reme4/reme.py
new file mode 100644
index 00000000..fe74bb7a
--- /dev/null
+++ b/reme4/reme.py
@@ -0,0 +1,43 @@
+"""ReMe memory management application entry point."""
+
+import asyncio
+import sys
+
+from .application import Application
+from .components import R
+from .config import parse_args, resolve_app_config
+from .enumeration import ComponentEnum
+from .utils import cli_find_reme, load_env, precheck_start
+
+
+class ReMe(Application):
+ """ReMe memory management application."""
+
+
+async def call_server(action: str, **kwargs):
+ """Call the appropriate server component."""
+ backend: str = kwargs.pop("backend", "http")
+ client_cls = R.get(ComponentEnum.CLIENT, backend)
+ async with client_cls(action=action, **kwargs) as client:
+ async for chunk in client():
+ print(chunk, end="", flush=True)
+ print()
+
+
+def main():
+ """Parse CLI arguments and launch the appropriate mode."""
+ action, kwargs = parse_args(*sys.argv[1:])
+ if action == "start":
+ load_env()
+ kwargs = resolve_app_config(**kwargs)
+ if not precheck_start(kwargs.get("service")):
+ return
+ ReMe(**kwargs).run_app()
+ elif action == "find_reme":
+ cli_find_reme()
+ else:
+ asyncio.run(call_server(action, **kwargs))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/reme4/schema/__init__.py b/reme4/schema/__init__.py
new file mode 100644
index 00000000..0a54891b
--- /dev/null
+++ b/reme4/schema/__init__.py
@@ -0,0 +1,25 @@
+"""Schema"""
+
+from .application_config import ApplicationConfig, ComponentConfig, JobConfig
+from .emb_node import EmbNode
+from .file_chunk import FileChunk
+from .file_front_matter import FileFrontMatter
+from .file_link import FileLink
+from .file_node import FileNode
+from .request import Request
+from .response import Response
+from .stream_chunk import StreamChunk
+
+__all__ = [
+ "ApplicationConfig",
+ "ComponentConfig",
+ "JobConfig",
+ "EmbNode",
+ "FileChunk",
+ "FileFrontMatter",
+ "FileLink",
+ "FileNode",
+ "Request",
+ "Response",
+ "StreamChunk",
+]
diff --git a/reme4/schema/application_config.py b/reme4/schema/application_config.py
new file mode 100644
index 00000000..a1d78c5f
--- /dev/null
+++ b/reme4/schema/application_config.py
@@ -0,0 +1,45 @@
+"""Application configuration models."""
+
+import os
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from ..enumeration import ComponentEnum
+
+
+class ComponentConfig(BaseModel):
+ """Base config for a component; extra fields allowed for backend-specific options."""
+
+ model_config = ConfigDict(extra="allow")
+
+ backend: str = Field(default="", description="Backend implementation class name")
+
+
+class JobConfig(ComponentConfig):
+ """Config for a job — an ordered sequence of step components."""
+
+ name: str = Field(default="", description="Unique job identifier")
+ description: str = Field(default="", description="Human-readable description")
+ parameters: dict = Field(default_factory=dict, description="Job-level parameters")
+ steps: list[ComponentConfig] = Field(default_factory=list, description="Ordered step configs")
+
+
+class ApplicationConfig(BaseModel):
+ """Root config for the ReMe application."""
+
+ app_name: str = Field(default=os.getenv("APP_NAME", "ReMe"), description="Application display name")
+ working_dir: str = Field(default=".reme", description="Working directory for runtime files")
+ metadata_dir: str = Field(default="reme_metadata", description="Subdirectory for ReMe persistent state")
+ daily_dir: str = Field(default="daily", description="Subdirectory for daily memory")
+ knowledge_dir: str = Field(default="knowledge", description="Subdirectory for knowledge")
+ enable_logo: bool = Field(default=True, description="Show ASCII logo on startup")
+ language: str = Field(default="", description="Default language for LLM interactions")
+ log_to_console: bool = Field(default=True, description="Log to console")
+ log_to_file: bool = Field(default=True, description="Log to file")
+ mcp_servers: dict[str, dict] = Field(default_factory=dict, description="MCP server configs by name")
+ service: ComponentConfig = Field(default_factory=ComponentConfig, description="Service endpoint config")
+ jobs: list[JobConfig] = Field(default_factory=list, description="Job definitions")
+ components: dict[ComponentEnum, dict[str, ComponentConfig]] = Field(
+ default_factory=dict,
+ description="Component registry keyed by type then name",
+ )
diff --git a/reme4/schema/emb_node.py b/reme4/schema/emb_node.py
new file mode 100644
index 00000000..2474a4e6
--- /dev/null
+++ b/reme4/schema/emb_node.py
@@ -0,0 +1,34 @@
+"""Embedding node — base record carrying text and its vector."""
+
+from uuid import uuid4
+
+import numpy as np
+from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator
+
+
+class EmbNode(BaseModel):
+ """A text record with an optional embedding vector and metadata."""
+
+ model_config = ConfigDict(arbitrary_types_allowed=True)
+
+ id: str = Field(default_factory=lambda: uuid4().hex, description="Unique node id")
+ text: str = Field(default="", description="Text content")
+ embedding: np.ndarray | None = Field(default=None, description="Embedding vector (float16)")
+ metadata: dict = Field(default_factory=dict, description="Arbitrary metadata")
+
+ @field_validator("embedding", mode="before")
+ @classmethod
+ def validate_embedding(cls, v):
+ """Coerce list/tuple to float16 ndarray."""
+ # Coerce list/tuple inputs into a float16 ndarray for compact storage.
+ if v is None:
+ return v
+ return np.array(v, dtype=np.float16)
+
+ @field_serializer("embedding")
+ def serialize_embedding(self, v: np.ndarray | None, _info):
+ """Serialize ndarray to a JSON-friendly list."""
+ # ndarray is not JSON-serializable; emit a plain list.
+ if v is None:
+ return None
+ return v.tolist()
diff --git a/reme4/schema/file_chunk.py b/reme4/schema/file_chunk.py
new file mode 100644
index 00000000..f7de723b
--- /dev/null
+++ b/reme4/schema/file_chunk.py
@@ -0,0 +1,26 @@
+"""File chunk — an embedding node tied to a line range in a file."""
+
+from pydantic import Field
+
+from .emb_node import EmbNode
+
+
+class FileChunk(EmbNode):
+ """A chunk of a file with positional info and per-stage retrieval scores."""
+
+ path: str = Field(default="", description="Vault-relative file path")
+ start_line: int = Field(default=0, description="Inclusive start line (0-based)")
+ end_line: int = Field(default=0, description="Exclusive end line")
+ scores: dict[str, float] = Field(default_factory=dict, description="Retrieval scores keyed by stage")
+
+ @property
+ def score(self) -> float:
+ """Final aggregated score; 0.0 if not yet computed."""
+ return self.scores.get("score", 0.0)
+
+ def set_hash_id(self):
+ """Replace ``id`` with a deterministic hash of (path, range, text)."""
+ from ..utils import hash_text
+
+ self.id = hash_text(" ".join([self.path, str(self.start_line), str(self.end_line), self.text]))
+ return self
diff --git a/reme4/schema/file_front_matter.py b/reme4/schema/file_front_matter.py
new file mode 100644
index 00000000..16e81537
--- /dev/null
+++ b/reme4/schema/file_front_matter.py
@@ -0,0 +1,24 @@
+"""FileFrontMatter — parsed Markdown front matter."""
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class FileFrontMatter(BaseModel):
+ """Markdown front matter; unknown keys are preserved as extras."""
+
+ model_config = ConfigDict(extra="allow")
+
+ title: str = Field(default="", description="Document title")
+ description: str = Field(default="", description="Document description")
+ tags: list[str] | None = Field(default=None, description="Tags; None if absent")
+
+ @property
+ def model_extra(self) -> dict[str, Any] | None:
+ """Get extra fields set during validation.
+
+ Returns:
+ A dictionary of extra fields, or `None` if `config.extra` is not set to `"allow"`.
+ """
+ return self.__pydantic_extra__
diff --git a/reme4/schema/file_link.py b/reme4/schema/file_link.py
new file mode 100644
index 00000000..b029d5a9
--- /dev/null
+++ b/reme4/schema/file_link.py
@@ -0,0 +1,18 @@
+"""FileLink"""
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class FileLink(BaseModel):
+ """file link
+ [[target_path]]
+ [[target_path#target_anchor]]
+ predicate:: [[target_*]]
+ [predicate:: [[target_*]]]
+ """
+
+ model_config = ConfigDict(extra="forbid")
+ source_path: str = Field(default=..., description="source file path relative to working dir")
+ target_path: str = Field(default=..., description="target file path relative to working dir")
+ target_anchor: str | None = Field(default=None, description="Heading or block anchor (text after '#')")
+ predicate: str | None = Field(default=None, description="Dataview-style typed-link predicate")
diff --git a/reme4/schema/file_node.py b/reme4/schema/file_node.py
new file mode 100644
index 00000000..9be276f8
--- /dev/null
+++ b/reme4/schema/file_node.py
@@ -0,0 +1,16 @@
+"""File node — a file's metadata, links, and chunk references in the graph."""
+
+from pydantic import BaseModel, Field
+
+from .file_front_matter import FileFrontMatter
+from .file_link import FileLink
+
+
+class FileNode(BaseModel):
+ """A vault file as a graph node."""
+
+ path: str = Field(default=..., description="Vault-relative file path")
+ st_mtime: float = Field(default=..., description="Filesystem mtime (seconds)")
+ links: list[FileLink] = Field(default_factory=list, description="Outgoing wikilinks")
+ chunk_ids: list[str] = Field(default_factory=list, description="Owned FileChunk ids")
+ front_matter: FileFrontMatter = Field(default_factory=FileFrontMatter, description="Parsed front matter")
diff --git a/reme4/schema/request.py b/reme4/schema/request.py
new file mode 100644
index 00000000..c1d9b2ab
--- /dev/null
+++ b/reme4/schema/request.py
@@ -0,0 +1,11 @@
+"""Request schema for service endpoints."""
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class Request(BaseModel):
+ """Incoming service request; extra fields are allowed for endpoint-specific payloads."""
+
+ model_config = ConfigDict(extra="allow")
+
+ metadata: dict = Field(default_factory=dict, description="Request metadata for context")
diff --git a/reme4/schema/response.py b/reme4/schema/response.py
new file mode 100644
index 00000000..89cbce33
--- /dev/null
+++ b/reme4/schema/response.py
@@ -0,0 +1,15 @@
+"""Response schema for service endpoints and LLM calls."""
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class Response(BaseModel):
+ """Standard response envelope; extra fields allowed for endpoint-specific output."""
+
+ model_config = ConfigDict(extra="allow")
+
+ answer: str | Any = Field(default="", description="Response content or result data")
+ success: bool = Field(default=True, description="Whether the operation succeeded")
+ metadata: dict = Field(default_factory=dict, description="Additional context and diagnostics")
diff --git a/reme4/schema/stream_chunk.py b/reme4/schema/stream_chunk.py
new file mode 100644
index 00000000..b2fb0aef
--- /dev/null
+++ b/reme4/schema/stream_chunk.py
@@ -0,0 +1,14 @@
+"""Stream chunk schema for incremental responses (e.g. LLM streaming)."""
+
+from pydantic import BaseModel, Field
+
+from ..enumeration import ChunkEnum
+
+
+class StreamChunk(BaseModel):
+ """A single chunk in a streaming response sequence."""
+
+ chunk_type: ChunkEnum = Field(default=ChunkEnum.CONTENT, description="Type of chunk content")
+ chunk: str | dict | list = Field(default="", description="Chunk payload")
+ done: bool = Field(default=False, description="Whether this is the final chunk")
+ metadata: dict = Field(default_factory=dict, description="Chunk metadata")
diff --git a/reme4/steps/__init__.py b/reme4/steps/__init__.py
new file mode 100644
index 00000000..fa25273c
--- /dev/null
+++ b/reme4/steps/__init__.py
@@ -0,0 +1,9 @@
+"""steps"""
+
+from . import common
+from .base_step import BaseStep
+
+__all__ = [
+ "common",
+ "BaseStep",
+]
diff --git a/reme4/steps/base_step.py b/reme4/steps/base_step.py
new file mode 100644
index 00000000..a710f457
--- /dev/null
+++ b/reme4/steps/base_step.py
@@ -0,0 +1,140 @@
+"""Base step class for LLM workflow execution."""
+
+import copy
+from abc import abstractmethod, ABC
+from typing import TypeVar, TYPE_CHECKING
+
+from agentscope.formatter import FormatterBase
+from agentscope.model import ChatModelBase
+from agentscope.token import TokenCounterBase
+
+from ..components.embedding import BaseEmbeddingModel
+from ..components.file_parser import BaseFileParser
+from ..components.file_store import BaseFileStore
+from ..components.file_watcher import BaseFileWatcher
+from ..components.prompt_handler import PromptHandler
+from ..components.runtime_context import RuntimeContext
+from ..enumeration import ComponentEnum
+from ..utils import get_logger
+
+if TYPE_CHECKING:
+ from ..components import ApplicationContext
+
+T = TypeVar("T")
+
+
+class BaseStep(ABC):
+ """Composable unit of an LLM workflow."""
+
+ component_type = ComponentEnum.STEP
+
+ def __new__(cls, *args, **kwargs):
+ # Snapshot init args so copy() can rebuild an equivalent instance later.
+ instance = object.__new__(cls)
+ instance._init_args = copy.copy(args)
+ instance._init_kwargs = copy.copy(kwargs)
+ return instance
+
+ def __init__(
+ self,
+ name: str | None = None,
+ backend: str = "",
+ app_context: "ApplicationContext | None" = None,
+ language: str = "",
+ prompt_dict: dict[str, str] | None = None,
+ input_mapping: dict[str, str] | None = None,
+ output_mapping: dict[str, str] | None = None,
+ **kwargs,
+ ):
+ super().__init__()
+ self.name: str = name or self.__class__.__name__
+ self.backend: str = backend
+ self.app_context: "ApplicationContext | None" = app_context
+ self.language: str = language
+ self.input_mapping = input_mapping
+ self.output_mapping = output_mapping
+ self.kwargs: dict = kwargs
+ self.context: RuntimeContext | None = None
+
+ self.logger = get_logger()
+ if hasattr(self.logger, "bind"):
+ self.logger = self.logger.bind(component=self.name)
+
+ # Load class-level prompts first, then overlay caller-provided overrides.
+ self.prompt = PromptHandler(language=self.language)
+ self.prompt.load_prompt_by_class(self.__class__).load_prompt_dict(prompt_dict)
+
+ @abstractmethod
+ async def execute(self):
+ """Run the step's logic against ``self.context``."""
+
+ async def __call__(self, context: RuntimeContext | None = None, **kwargs):
+ # Build runtime context, then apply key remapping around execute().
+ self.context = RuntimeContext.from_context(context, **kwargs)
+ assert self.context is not None
+ if self.input_mapping:
+ self.context.apply_mapping(self.input_mapping)
+ result = await self.execute()
+ if self.output_mapping:
+ self.context.apply_mapping(self.output_mapping)
+ return result
+
+ def _resolve(self, key: str, base_cls: type[T], comp_enum: ComponentEnum, attr: str | None = None) -> T:
+ """Return a kwargs-supplied instance, or look one up by name in the app registry."""
+ # 1. Step init kwargs, 2. Runtime context (run_job kwargs), 3. App registry by name.
+ for source in (self.kwargs, self.context or {}):
+ value = source.get(key)
+ if isinstance(value, base_cls):
+ return value
+
+ name = self.kwargs.get(key, "default")
+ assert self.app_context is not None
+ comp = self.app_context.components[comp_enum][name]
+ return getattr(comp, attr) if attr else comp
+
+ @property
+ def as_llm(self) -> ChatModelBase:
+ """Return the chat model component."""
+ return self._resolve("as_llm", ChatModelBase, ComponentEnum.AS_LLM, "model")
+
+ @property
+ def as_llm_formatter(self) -> FormatterBase:
+ """Return the LLM formatter component."""
+ return self._resolve("as_llm_formatter", FormatterBase, ComponentEnum.AS_LLM_FORMATTER, "formatter")
+
+ @property
+ def as_token_counter(self) -> TokenCounterBase:
+ """Return the token counter component."""
+ return self._resolve("as_token_counter", TokenCounterBase, ComponentEnum.AS_TOKEN_COUNTER, "token_counter")
+
+ @property
+ def file_parser(self) -> BaseFileParser:
+ """Return the file parser component."""
+ return self._resolve("file_parser", BaseFileParser, ComponentEnum.FILE_PARSER)
+
+ @property
+ def file_store(self) -> BaseFileStore:
+ """Return the file store component."""
+ return self._resolve("file_store", BaseFileStore, ComponentEnum.FILE_STORE)
+
+ @property
+ def embedding(self) -> BaseEmbeddingModel:
+ """Return the embedding model component."""
+ return self._resolve("embedding", BaseEmbeddingModel, ComponentEnum.EMBEDDING_MODEL)
+
+ @property
+ def file_watcher(self) -> BaseFileWatcher:
+ """Return the file watcher component."""
+ return self._resolve("file_watcher", BaseFileWatcher, ComponentEnum.FILE_WATCHER)
+
+ def prompt_format(self, prompt_name: str, **kwargs) -> str:
+ """Format a named prompt template with the given kwargs."""
+ return self.prompt.prompt_format(prompt_name=prompt_name, **kwargs)
+
+ def get_prompt(self, prompt_name: str) -> str:
+ """Return a named prompt template as-is."""
+ return self.prompt.get_prompt(prompt_name=prompt_name)
+
+ def copy(self, **kwargs) -> "BaseStep":
+ """Construct a new instance from the original init args, applying overrides."""
+ return self.__class__(*self._init_args, **{**self._init_kwargs, **kwargs})
diff --git a/reme4/steps/common/__init__.py b/reme4/steps/common/__init__.py
new file mode 100644
index 00000000..e09c2073
--- /dev/null
+++ b/reme4/steps/common/__init__.py
@@ -0,0 +1,21 @@
+"""Common steps."""
+
+from .demo import DemoEchoStep1, DemoEchoStep2
+from .health_check import HealthCheckStep
+from .help import HelpStep
+from .reindex import ReindexStep
+from .search import SearchStep
+from .stream_demo import StreamDemoStep1, StreamDemoStep2
+from .version import VersionStep
+
+__all__ = [
+ "DemoEchoStep1",
+ "DemoEchoStep2",
+ "HealthCheckStep",
+ "HelpStep",
+ "ReindexStep",
+ "SearchStep",
+ "StreamDemoStep1",
+ "StreamDemoStep2",
+ "VersionStep",
+]
diff --git a/reme4/steps/common/demo.py b/reme4/steps/common/demo.py
new file mode 100644
index 00000000..e7c8df1a
--- /dev/null
+++ b/reme4/steps/common/demo.py
@@ -0,0 +1,53 @@
+"""Demo steps for smoke-testing the application stack."""
+
+from ..base_step import BaseStep
+from ...components import R
+
+
+@R.register("demo_echo_step1")
+class DemoEchoStep1(BaseStep):
+ """Read query/min_score from context, normalize, and write back for Step2."""
+
+ async def execute(self):
+ assert self.context is not None
+ query = self.context.get("query", "")
+ min_score = self.context.get("min_score", 0.5)
+
+ self.logger.info(f"[{self.name}] query={query!r}, min_score={min_score}")
+
+ processed_query = query.strip().lower()
+ adjusted_min_score = float(min_score) * 0.9
+
+ self.context["processed_query"] = processed_query
+ self.context["adjusted_min_score"] = adjusted_min_score
+
+ return self.context.response
+
+
+@R.register("demo_echo_step2")
+class DemoEchoStep2(BaseStep):
+ """Consume Step1's outputs from context and emit the final response."""
+
+ async def execute(self):
+ assert self.context is not None
+ query = self.context.get("query", "")
+ min_score = self.context.get("min_score", 0.5)
+ processed_query = self.context.get("processed_query", "")
+ adjusted_min_score = self.context.get("adjusted_min_score", min_score)
+
+ self.logger.info(
+ f"[{self.name}] query={query!r}, min_score={min_score}, "
+ f"processed_query={processed_query!r}, adjusted_min_score={adjusted_min_score}",
+ )
+
+ self.context.response.answer = f"echo: {processed_query} (min_score={adjusted_min_score})"
+ self.context.response.metadata.update(
+ {
+ "step": self.name,
+ "query": query,
+ "min_score": min_score,
+ "processed_query": processed_query,
+ "adjusted_min_score": adjusted_min_score,
+ },
+ )
+ return self.context.response
diff --git a/reme4/steps/common/health_check.py b/reme4/steps/common/health_check.py
new file mode 100644
index 00000000..6fc72652
--- /dev/null
+++ b/reme4/steps/common/health_check.py
@@ -0,0 +1,164 @@
+"""Return a concise health check snapshot of ReMe runtime components."""
+
+import sys
+from collections.abc import Mapping
+
+import numpy as np
+
+from ..base_step import BaseStep
+from ... import __version__
+from ...components import R
+from ...enumeration import ComponentEnum
+
+
+def _deep_size(obj, _seen: set | None = None) -> int:
+ """Recursive sizeof; uses ndarray.nbytes for numpy and walks containers / __dict__."""
+ if _seen is None:
+ _seen = set()
+ obj_id = id(obj)
+ if obj_id in _seen:
+ return 0
+ _seen.add(obj_id)
+
+ if isinstance(obj, np.ndarray):
+ return int(obj.nbytes) + sys.getsizeof(obj)
+
+ size = sys.getsizeof(obj)
+ if isinstance(obj, (str, bytes, bytearray, int, float, bool, type(None))):
+ return size
+ if isinstance(obj, Mapping):
+ size += sum(_deep_size(k, _seen) + _deep_size(v, _seen) for k, v in obj.items())
+ elif isinstance(obj, (list, tuple, set, frozenset)):
+ size += sum(_deep_size(item, _seen) for item in obj)
+ elif hasattr(obj, "__dict__"):
+ size += _deep_size(vars(obj), _seen)
+ elif hasattr(obj, "__slots__"):
+ for slot in obj.__slots__:
+ if hasattr(obj, slot):
+ size += _deep_size(getattr(obj, slot), _seen)
+ return size
+
+
+def _mb_str(*objs) -> str:
+ """Return summed deep size of objs formatted as 'X.XX MB'."""
+ seen: set = set()
+ total = sum(_deep_size(o, seen) for o in objs)
+ return f"{total / (1024 * 1024):.2f} MB"
+
+
+def _embedding_status(comp) -> dict:
+ return {
+ "is_started": comp.is_started,
+ "is_healthy": getattr(comp, "is_healthy", None),
+ "model_name": getattr(comp, "model_name", None),
+ "dimensions": getattr(comp, "dimensions", None),
+ "cache_size": len(getattr(comp, "_embedding_cache", {}) or {}),
+ "memory": _mb_str(getattr(comp, "_embedding_cache", {}) or {}),
+ }
+
+
+def _file_graph_status(comp) -> dict:
+ # Nx backend: single _graph attribute holds nodes/edges, virtuals are nodes without "node" payload.
+ g = getattr(comp, "_graph", None)
+ if g is not None:
+ n_real = sum(1 for _, d in g.nodes(data=True) if "node" in d)
+ return {
+ "is_started": comp.is_started,
+ "n_nodes": n_real,
+ "n_edges": g.number_of_edges(),
+ "n_virtual": g.number_of_nodes() - n_real,
+ "memory": _mb_str(g),
+ }
+ # Local backend: separate dicts for nodes, resolved inverse edges, and pending edges.
+ nodes = getattr(comp, "_nodes", {}) or {}
+ inverse = getattr(comp, "_inverse", {}) or {}
+ pending = getattr(comp, "_pending", {}) or {}
+ return {
+ "is_started": comp.is_started,
+ "n_nodes": len(nodes),
+ "n_edges": sum(len(s) for s in inverse.values()),
+ "n_pending": sum(len(s) for s in pending.values()),
+ "memory": _mb_str(nodes, inverse, pending),
+ }
+
+
+def _file_store_status(comp) -> dict:
+ chunks = getattr(comp, "file_chunks", {}) or {}
+ return {
+ "is_started": comp.is_started,
+ "n_chunks": len(chunks),
+ "n_chunks_with_embedding": sum(1 for c in chunks.values() if getattr(c, "embedding", None) is not None),
+ "memory": _mb_str(chunks),
+ }
+
+
+def _file_watcher_status(comp) -> dict:
+ bg = getattr(comp, "_background_task", None)
+ return {
+ "is_started": comp.is_started,
+ "background_running": bool(bg and not bg.done()),
+ "watch_paths": [str(p) for p in (getattr(comp, "watch_paths", []) or [])],
+ }
+
+
+def _keyword_index_status(comp) -> dict:
+ return {
+ "is_started": comp.is_started,
+ "n_docs": getattr(comp, "n_docs", None),
+ "vocab_size": len(getattr(comp, "vocab", {}) or {}),
+ "memory": _mb_str(
+ getattr(comp, "vocab", {}) or {},
+ getattr(comp, "inverted_index", {}) or {},
+ getattr(comp, "doc_meta", {}) or {},
+ getattr(comp, "_idf_cache", {}) or {},
+ ),
+ }
+
+
+_HANDLERS = {
+ ComponentEnum.EMBEDDING_MODEL: _embedding_status,
+ ComponentEnum.FILE_GRAPH: _file_graph_status,
+ ComponentEnum.FILE_STORE: _file_store_status,
+ ComponentEnum.FILE_WATCHER: _file_watcher_status,
+ ComponentEnum.KEYWORD_INDEX: _keyword_index_status,
+}
+
+
+def _is_status_healthy(ctype: ComponentEnum, status: dict) -> bool:
+ """Per-component health rule. Unstarted = unhealthy; type-specific extras checked."""
+ if not status.get("is_started"):
+ return False
+ if ctype is ComponentEnum.EMBEDDING_MODEL and status.get("is_healthy") is False:
+ return False
+ if ctype is ComponentEnum.FILE_WATCHER and not status.get("background_running"):
+ return False
+ return True
+
+
+@R.register("health_check_step")
+class HealthCheckStep(BaseStep):
+ """Collect a concise health check snapshot of the relevant components."""
+
+ async def execute(self):
+ assert self.context is not None
+
+ components: dict = {}
+ healthy = True
+ if self.app_context is not None:
+ for ctype, handler in _HANDLERS.items():
+ comp_map = self.app_context.components.get(ctype, {})
+ bucket = {}
+ for name, comp in comp_map.items():
+ s = handler(comp)
+ bucket[name] = s
+ if not _is_status_healthy(ctype, s):
+ healthy = False
+ components[ctype.value] = bucket
+
+ health = {"version": __version__, "healthy": healthy, "components": components}
+ self.logger.info(f"[{self.name}] health collected: {health}")
+
+ status_emoji = "✅" if healthy else "❌"
+ self.context.response.answer = f"{status_emoji} ReMe v{__version__} - {'healthy' if healthy else 'unhealthy'}"
+ self.context.response.metadata["health"] = health
+ return self.context.response
diff --git a/reme4/steps/common/help.py b/reme4/steps/common/help.py
new file mode 100644
index 00000000..6c850772
--- /dev/null
+++ b/reme4/steps/common/help.py
@@ -0,0 +1,42 @@
+"""Return a one-line summary of every registered job for LLM consumption."""
+
+from ..base_step import BaseStep
+from ...components import R
+
+
+@R.register("help_step")
+class HelpStep(BaseStep):
+ """List all registered jobs (excluding self) as compact one-liners for an LLM."""
+
+ @staticmethod
+ def _format_params(parameters: dict) -> str:
+ props = (parameters or {}).get("properties") or {}
+ if not props:
+ return "no args"
+ required = set((parameters or {}).get("required") or [])
+ parts = []
+ for pname, pschema in props.items():
+ ptype = pschema.get("type", "any")
+ if pname in required:
+ parts.append(f"{pname}:{ptype}*")
+ elif "default" in pschema:
+ parts.append(f"{pname}:{ptype}={pschema['default']}")
+ else:
+ parts.append(f"{pname}:{ptype}")
+ return ", ".join(parts)
+
+ async def execute(self):
+ assert self.context is not None
+
+ lines = []
+ if self.app_context is not None:
+ for name, job in self.app_context.jobs.items():
+ if name == "help":
+ continue
+ lines.append(f"🛠️ `{name}` — {job.description} 📥 {self._format_params(job.parameters)}")
+
+ self.logger.info(f"[{self.name}] returning {len(lines)} jobs")
+
+ self.context.response.answer = "\n".join(lines)
+ self.context.response.metadata["job_count"] = len(lines)
+ return self.context.response
diff --git a/reme4/steps/common/reindex.py b/reme4/steps/common/reindex.py
new file mode 100644
index 00000000..f4d280e9
--- /dev/null
+++ b/reme4/steps/common/reindex.py
@@ -0,0 +1,24 @@
+"""Wipe the file store and rebuild it from the watcher's tracked files."""
+
+from ..base_step import BaseStep
+from ...components import R
+
+
+@R.register("reindex_step")
+class ReindexStep(BaseStep):
+ """Full re-index: stop watcher, clear store, sync from disk, then restart."""
+
+ async def execute(self):
+ assert self.context is not None
+
+ await self.file_watcher.close()
+ try:
+ await self.file_store.clear()
+ counts = await self.file_watcher.update_store()
+ finally:
+ await self.file_watcher.start()
+
+ self.logger.info(f"[{self.name}] reindexed {counts}")
+ self.context.response.answer = f"🔄 Reindexed {counts['added']} file(s)"
+ self.context.response.metadata["counts"] = counts
+ return self.context.response
diff --git a/reme4/steps/common/search.py b/reme4/steps/common/search.py
new file mode 100644
index 00000000..595d665e
--- /dev/null
+++ b/reme4/steps/common/search.py
@@ -0,0 +1,227 @@
+"""Hybrid search over file_store using RRF fusion of vector + keyword results."""
+
+import asyncio
+
+from ..base_step import BaseStep
+from ...components import R
+from ...schema import FileChunk, FileLink, FileNode
+
+_RRF_K = 60
+_MAX_CANDIDATES = 200
+
+
+@R.register("search_step")
+class SearchStep(BaseStep):
+ """Hybrid search: run vector + keyword in parallel, fuse via RRF, filter, truncate."""
+
+ @staticmethod
+ def _rrf_merge(
+ vector: list[FileChunk],
+ keyword: list[FileChunk],
+ vector_weight: float,
+ ) -> list[FileChunk]:
+ """Fuse two ranked lists with Reciprocal Rank Fusion, keyed by chunk.id."""
+ text_weight = 1.0 - vector_weight
+ merged: dict[str, FileChunk] = {}
+
+ for rank, chunk in enumerate(vector, start=1):
+ contrib = vector_weight / (_RRF_K + rank)
+ c = chunk.model_copy(deep=False)
+ c.scores = {**chunk.scores, "vector": chunk.scores.get("vector", chunk.score), "score": contrib}
+ merged[c.id] = c
+
+ for rank, chunk in enumerate(keyword, start=1):
+ contrib = text_weight / (_RRF_K + rank)
+ existing = merged.get(chunk.id)
+ if existing is not None:
+ existing.scores = {
+ **existing.scores,
+ "keyword": chunk.scores.get("keyword", chunk.score),
+ "score": existing.scores["score"] + contrib,
+ }
+ else:
+ c = chunk.model_copy(deep=False)
+ c.scores = {**chunk.scores, "keyword": chunk.scores.get("keyword", chunk.score), "score": contrib}
+ merged[c.id] = c
+
+ results = list(merged.values())
+ results.sort(key=lambda r: r.score, reverse=True)
+ return results
+
+ @staticmethod
+ def _format_scores(scores: dict[str, float], hybrid: bool) -> str:
+ """Format scores for the answer line: always show fused; show per-branch when hybrid."""
+ parts = [f"score={scores.get('score', 0.0):.4f}"]
+ if hybrid:
+ for k in ("vector", "keyword"):
+ v = scores.get(k)
+ parts.append(f"{k}={v:.4f}" if v is not None else f"{k}=-")
+ return " ".join(parts)
+
+ @staticmethod
+ def _group_by_neighbor(links: list[FileLink], key_attr: str) -> dict[str, list[dict]]:
+ """Group edges by neighbor path (insertion-ordered), each value a list of {predicate, anchor}."""
+ out: dict[str, list[dict]] = {}
+ for lnk in links:
+ neighbor = getattr(lnk, key_attr)
+ if not neighbor:
+ continue
+ out.setdefault(neighbor, []).append(
+ {"predicate": lnk.predicate, "anchor": lnk.target_anchor},
+ )
+ return out
+
+ @staticmethod
+ def _node_meta(node: FileNode | None) -> dict:
+ """Extract a compact meta dict (title/description/tags) from a FileNode."""
+ if node is None:
+ return {}
+ fm = node.front_matter
+ meta: dict = {}
+ if fm.title:
+ meta["title"] = fm.title
+ if fm.description:
+ meta["description"] = fm.description
+ if fm.tags:
+ meta["tags"] = list(fm.tags)
+ return meta
+
+ @staticmethod
+ def _format_meta_inline(meta: dict) -> str:
+ """One-line render of node meta for the answer; '(no meta)' when empty."""
+ parts = []
+ if "title" in meta:
+ parts.append(f'title="{meta["title"]}"')
+ if "tags" in meta:
+ parts.append(f"tags={meta['tags']}")
+ return " ".join(parts) if parts else "(no meta)"
+
+ @staticmethod
+ def _format_via(edge: dict) -> str:
+ """Render a single (predicate, anchor) edge as a 'via ...' descriptor."""
+ bits = []
+ if edge.get("predicate"):
+ bits.append(f"predicate={edge['predicate']}")
+ if edge.get("anchor"):
+ bits.append(f"anchor=#{edge['anchor']}")
+ return ", ".join(bits) if bits else "plain"
+
+ async def _expand_links(
+ self,
+ chunk_paths: list[str],
+ max_per_direction: int,
+ ) -> dict[str, dict]:
+ """Fetch out/in links for each chunk path; attach neighbor meta. Returns per-path expansion."""
+ if not chunk_paths:
+ return {}
+
+ out_lists, in_lists = await asyncio.gather(
+ asyncio.gather(*(self.file_store.get_outlinks(p) for p in chunk_paths)),
+ asyncio.gather(*(self.file_store.get_inlinks(p) for p in chunk_paths)),
+ )
+
+ # Pre-group + cap per direction so we only fetch meta for displayed neighbors.
+ out_grouped = [
+ dict(list(self._group_by_neighbor(outs, "target_path").items())[:max_per_direction]) for outs in out_lists
+ ]
+ in_grouped = [
+ dict(list(self._group_by_neighbor(ins, "source_path").items())[:max_per_direction]) for ins in in_lists
+ ]
+
+ neighbor_paths = sorted({n for g in out_grouped for n in g} | {n for g in in_grouped for n in g})
+ nodes = await self.file_store.get_nodes(neighbor_paths) if neighbor_paths else []
+ meta_by_path = {n.path: self._node_meta(n) for n in nodes}
+
+ def _attach(grouped: dict[str, list[dict]]) -> list[dict]:
+ return [
+ {"path": npath, "meta": meta_by_path.get(npath, {}), "edges": edges} for npath, edges in grouped.items()
+ ]
+
+ return {
+ cp: {"outlinks": _attach(og), "inlinks": _attach(ig)}
+ for cp, og, ig in zip(chunk_paths, out_grouped, in_grouped)
+ }
+
+ @classmethod
+ def _render_expansion_lines(cls, expansion: dict) -> list[str]:
+ """Render outlinks/inlinks blocks for one chunk path; return zero or more indented lines."""
+ lines: list[str] = []
+ for direction, arrow, items in (
+ ("outlinks", "→", expansion.get("outlinks") or []),
+ ("inlinks", "←", expansion.get("inlinks") or []),
+ ):
+ if not items:
+ continue
+ lines.append(f" {direction} ({len(items)}):")
+ for item in items:
+ lines.append(f" {arrow} {item['path']} {cls._format_meta_inline(item['meta'])}")
+ for edge in item["edges"]:
+ lines.append(f" via {cls._format_via(edge)}")
+ return lines
+
+ async def execute(self):
+ assert self.context is not None
+ query: str = (self.context.get("query", "") or "").strip()
+ limit: int = int(self.context.get("limit", 5))
+ min_score: float = float(self.context.get("min_score", 0.0))
+ vector_weight: float = float(self.context.get("vector_weight", 0.7))
+ candidate_multiplier: float = float(self.context.get("candidate_multiplier", 3.0))
+ expand_links: bool = bool(self.context.get("expand_links", True))
+ max_links_per_direction: int = int(self.context.get("max_links_per_direction", 10))
+
+ assert query, "query cannot be empty"
+ assert 0.0 <= vector_weight <= 1.0, f"vector_weight must be in [0, 1], got {vector_weight}"
+ assert limit > 0, f"limit must be positive, got {limit}"
+
+ candidates = min(_MAX_CANDIDATES, max(1, int(limit * candidate_multiplier)))
+ search_filter: dict = self.context.get("search_filter", {}) or {}
+
+ vector_results, keyword_results = await asyncio.gather(
+ self.file_store.vector_search(query, candidates, search_filter),
+ self.file_store.keyword_search(query, candidates, search_filter),
+ )
+
+ self.logger.info(
+ f"[{self.name}] query={query!r} candidates={candidates} "
+ f"vector_hits={len(vector_results)} keyword_hits={len(keyword_results)}",
+ )
+
+ hybrid = bool(vector_results) and bool(keyword_results)
+ if not vector_results and not keyword_results:
+ fused: list[FileChunk] = []
+ elif not keyword_results:
+ fused = vector_results
+ elif not vector_results:
+ fused = keyword_results
+ else:
+ fused = self._rrf_merge(vector_results, keyword_results, vector_weight)
+
+ if min_score > 0.0:
+ fused = [c for c in fused if c.score >= min_score]
+ fused = fused[:limit]
+
+ unique_paths = list(dict.fromkeys(c.path for c in fused))
+ link_expansion: dict[str, dict] = (
+ await self._expand_links(unique_paths, max_links_per_direction) if expand_links else {}
+ )
+
+ answer_lines: list[str] = []
+ for c in fused:
+ answer_lines.append(
+ f"========== {c.path}:{c.start_line}-{c.end_line} "
+ f"[{self._format_scores(c.scores, hybrid)}] ==========\n{c.text}",
+ )
+ answer_lines.extend(self._render_expansion_lines(link_expansion.get(c.path, {})))
+
+ self.context.response.answer = "\n".join(answer_lines)
+ self.context.response.metadata["results"] = [
+ c.model_dump(exclude_none=True, exclude={"embedding"}) for c in fused
+ ]
+ self.context.response.metadata["link_expansion"] = link_expansion
+ self.context.response.metadata["counts"] = {
+ "vector": len(vector_results),
+ "keyword": len(keyword_results),
+ "returned": len(fused),
+ "hybrid": hybrid,
+ }
+ return self.context.response
diff --git a/reme4/steps/common/stream_demo.py b/reme4/steps/common/stream_demo.py
new file mode 100644
index 00000000..450077aa
--- /dev/null
+++ b/reme4/steps/common/stream_demo.py
@@ -0,0 +1,42 @@
+"""Streaming demo steps: step1 prepares text, step2 streams it char-by-char."""
+
+import asyncio
+
+from ..base_step import BaseStep
+from ...components import R
+from ...enumeration import ChunkEnum
+
+
+@R.register("stream_demo_step1")
+class StreamDemoStep1(BaseStep):
+ """Read query from context, repeat it 10x, write back for Step2 to stream."""
+
+ async def execute(self):
+ assert self.context is not None
+ query = self.context.get("query", "")
+ repeat = int(self.context.get("repeat", 10))
+
+ stream_text = (query * repeat) if query else ""
+
+ self.logger.info(f"[{self.name}] query={query!r}, repeat={repeat}, len={len(stream_text)}")
+
+ self.context["stream_text"] = stream_text
+ return self.context.response
+
+
+@R.register("stream_demo_step2")
+class StreamDemoStep2(BaseStep):
+ """Stream stream_text char-by-char as CONTENT chunks with 0.1s pacing."""
+
+ async def execute(self):
+ assert self.context is not None
+ stream_text: str = self.context.get("stream_text", "")
+ interval = float(self.context.get("interval", 0.1))
+
+ self.logger.info(f"[{self.name}] streaming {len(stream_text)} chars, interval={interval}s")
+
+ for ch in stream_text:
+ await self.context.add_stream_string(ch, ChunkEnum.CONTENT)
+ await asyncio.sleep(interval)
+
+ return self.context.response
diff --git a/reme4/steps/common/version.py b/reme4/steps/common/version.py
new file mode 100644
index 00000000..e43aa698
--- /dev/null
+++ b/reme4/steps/common/version.py
@@ -0,0 +1,19 @@
+"""Return the package version."""
+
+from ..base_step import BaseStep
+
+from ...components import R
+
+
+@R.register("version_step")
+class VersionStep(BaseStep):
+ """Emit reme4.__version__ as the response answer."""
+
+ async def execute(self):
+ assert self.context is not None
+ from ... import __version__
+
+ self.logger.info(f"[{self.name}] version={__version__}")
+ self.context.response.answer = __version__
+ self.context.response.metadata["version"] = __version__
+ return self.context.response
diff --git a/reme4/utils/__init__.py b/reme4/utils/__init__.py
new file mode 100644
index 00000000..45a5bfb9
--- /dev/null
+++ b/reme4/utils/__init__.py
@@ -0,0 +1,31 @@
+"""Utility modules."""
+
+from .common_utils import (
+ hash_text,
+ execute_stream_task,
+ mock_reme_server,
+ call_action,
+ call_and_check,
+)
+from .env_utils import load_env
+from .logger_utils import get_logger
+from .logo_utils import print_logo
+from .service_utils import find_reme, locate_reme, precheck_start, cli_find_reme
+from .similarity_utils import cosine_similarity, batch_cosine_similarity
+
+__all__ = [
+ "hash_text",
+ "execute_stream_task",
+ "mock_reme_server",
+ "call_action",
+ "call_and_check",
+ "load_env",
+ "get_logger",
+ "print_logo",
+ "find_reme",
+ "locate_reme",
+ "precheck_start",
+ "cli_find_reme",
+ "cosine_similarity",
+ "batch_cosine_similarity",
+]
diff --git a/reme4/utils/common_utils.py b/reme4/utils/common_utils.py
new file mode 100644
index 00000000..7df0dbef
--- /dev/null
+++ b/reme4/utils/common_utils.py
@@ -0,0 +1,249 @@
+"""Common utilities: hashing and async stream task execution."""
+
+import asyncio
+import hashlib
+import json
+import socket
+import subprocess
+import sys
+import time
+from collections.abc import AsyncGenerator, Callable
+from contextlib import asynccontextmanager
+from typing import Any, Literal
+
+from .logger_utils import get_logger
+from ..constants import REME_DEFAULT_HOST, REME_DEFAULT_PORT
+from ..enumeration import ChunkEnum
+from ..schema import StreamChunk
+
+
+def hash_text(text: str, encoding: str = "utf-8") -> str:
+ """Return SHA-256 hex digest of text."""
+ return hashlib.sha256(text.encode(encoding)).hexdigest()
+
+
+def _format_chunk(
+ chunk: StreamChunk,
+ output_format: Literal["str", "bytes", "chunk"],
+) -> str | bytes | StreamChunk:
+ """Render a StreamChunk in the requested transport format."""
+ if output_format == "chunk":
+ return chunk
+ data = "data:[DONE]\n\n" if chunk.done else f"data:{chunk.model_dump_json()}\n\n"
+ return data.encode() if output_format == "bytes" else data
+
+
+async def execute_stream_task(
+ stream_queue: asyncio.Queue[StreamChunk],
+ task: asyncio.Task[Any],
+ task_name: str | None = None,
+ output_format: Literal["str", "bytes", "chunk"] = "str",
+) -> AsyncGenerator[str | bytes | StreamChunk, None]:
+ """Yield chunks from stream_queue while monitoring task; cancels task on exit.
+
+ output_format: "str"/"bytes" emit SSE frames, "chunk" emits raw StreamChunk.
+ """
+ logger = get_logger()
+ consumer: asyncio.Task[StreamChunk] | None = None
+ try:
+ while True:
+ consumer = get_chunk = asyncio.create_task(stream_queue.get())
+ done, _pending = await asyncio.wait({get_chunk, task}, return_when=asyncio.FIRST_COMPLETED)
+
+ # Producer still running — relay the next chunk and continue.
+ if task not in done:
+ chunk = get_chunk.result()
+ yield _format_chunk(chunk, output_format)
+ if chunk.done:
+ return
+ continue
+
+ # Producer finished. Capture any pending chunk, then stop the consumer wait
+ # so we can inspect task state safely.
+ pending_chunk: StreamChunk | None = None
+ if get_chunk in done:
+ pending_chunk = get_chunk.result()
+ else:
+ get_chunk.cancel()
+ try:
+ await get_chunk
+ except asyncio.CancelledError:
+ pass
+
+ # Surface task failure first — an exception trumps trailing data.
+ if task.cancelled():
+ msg = f"Task cancelled: {task_name}" if task_name else "Task cancelled"
+ raise asyncio.CancelledError(msg)
+ exc = task.exception()
+ if exc is not None:
+ log_msg = f"Task error in {task_name}: {exc}" if task_name else f"Task error: {exc}"
+ logger.error(log_msg, exc_info=exc)
+ raise exc
+
+ # Producer ended cleanly — flush pending + drain queue so no chunk is lost,
+ # then emit the terminal sentinel.
+ if pending_chunk is not None:
+ yield _format_chunk(pending_chunk, output_format)
+ if pending_chunk.done:
+ return
+ while not stream_queue.empty():
+ chunk = stream_queue.get_nowait()
+ yield _format_chunk(chunk, output_format)
+ if chunk.done:
+ return
+
+ yield _format_chunk(StreamChunk(chunk_type=ChunkEnum.DONE, chunk="", done=True), output_format)
+ return
+
+ finally:
+ # Cancel consumer wait if still pending (e.g. on consumer aclose).
+ if consumer is not None and not consumer.done():
+ consumer.cancel()
+ try:
+ await consumer
+ except asyncio.CancelledError:
+ pass
+ # Cancel producer task if still running to avoid resource leaks.
+ if not task.done():
+ task.cancel()
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+
+
+def _pick_free_port(host: str = REME_DEFAULT_HOST) -> int:
+ """Bind to port 0 and return the OS-assigned free port."""
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+ s.bind((host, 0))
+ return s.getsockname()[1]
+
+
+async def _wait_reme_ready(host: str, port: int, timeout: float) -> None:
+ """Poll find_reme until it reports 'reme' or timeout elapses."""
+ from .service_utils import find_reme
+
+ deadline = time.time() + timeout
+ while time.time() < deadline:
+ status = await find_reme(host, port)
+ if status == "reme":
+ return
+ await asyncio.sleep(0.2)
+ raise TimeoutError(f"ReMe service did not become ready at {host}:{port} within {timeout}s")
+
+
+@asynccontextmanager
+async def mock_reme_server(
+ host: str = REME_DEFAULT_HOST,
+ port: int | None = None,
+ config: str | None = None,
+ extra_args: list[str] | None = None,
+ startup_timeout: float = 30.0,
+ shutdown_timeout: float = 10.0,
+ log_to_file: bool = False,
+ enable_logo: bool = False,
+):
+ """Spawn `reme4 start` as a subprocess and yield (host, port) once ready.
+
+ Auto-picks a free port when port is None. Subprocess is terminated on exit.
+ """
+ logger = get_logger()
+ if port is None:
+ port = _pick_free_port(host)
+
+ cmd: list[str] = [
+ sys.executable,
+ "-m",
+ "reme4.reme",
+ "start",
+ f"service.host={host}",
+ f"service.port={port}",
+ f"log_to_file={'true' if log_to_file else 'false'}",
+ f"enable_logo={'true' if enable_logo else 'false'}",
+ ]
+ if config:
+ cmd.append(f"config={config}")
+ if extra_args:
+ cmd.extend(extra_args)
+
+ logger.info(f"Launching mock reme server: {' '.join(cmd)}")
+ proc = subprocess.Popen(
+ cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ text=True,
+ )
+ try:
+ await _wait_reme_ready(host, port, startup_timeout)
+ yield host, port
+ except Exception:
+ # Capture early-exit output for diagnostics.
+ if proc.poll() is not None and proc.stdout is not None:
+ tail = proc.stdout.read()
+ logger.error(f"reme server exited early. output:\n{tail}")
+ raise
+ finally:
+ if proc.poll() is None:
+ proc.terminate()
+ try:
+ proc.wait(timeout=shutdown_timeout)
+ except subprocess.TimeoutExpired:
+ logger.warning("reme server did not terminate gracefully, killing")
+ proc.kill()
+ proc.wait(timeout=shutdown_timeout)
+ if proc.stdout is not None:
+ try:
+ proc.stdout.close()
+ except Exception:
+ pass
+
+
+async def call_action(
+ action: str,
+ host: str = REME_DEFAULT_HOST,
+ port: int = REME_DEFAULT_PORT,
+ timeout: float = 30.0,
+ **kwargs,
+) -> dict | str:
+ """POST to /{action}; return parsed JSON (dict) for JSON endpoints, raw text for SSE."""
+ from ..components.client.http_client import HttpClient
+
+ pieces: list[str] = []
+ async with HttpClient(action=action, host=host, port=port, timeout=timeout, **kwargs) as client:
+ async for chunk in client.stream_chunks():
+ payload = chunk.chunk
+ pieces.append(payload if isinstance(payload, str) else json.dumps(payload, ensure_ascii=False))
+ raw = "".join(pieces)
+ try:
+ return json.loads(raw)
+ except (ValueError, json.JSONDecodeError):
+ return raw
+
+
+async def call_and_check(
+ action: str,
+ host: str = REME_DEFAULT_HOST,
+ port: int = REME_DEFAULT_PORT,
+ validator: Callable[[Any], bool] | None = None,
+ expected: Any = None,
+ timeout: float = 30.0,
+ **kwargs,
+) -> Any:
+ """Call action and verify response. Raises AssertionError on mismatch.
+
+ - validator(result) -> bool: custom predicate.
+ - expected: deep-equality target (compared to result, or to result[key] when expected is dict).
+ """
+ result = await call_action(action, host=host, port=port, timeout=timeout, **kwargs)
+ if validator is not None and not validator(result):
+ raise AssertionError(f"validator rejected response for action={action!r}: {result!r}")
+ if expected is not None:
+ if isinstance(expected, dict) and isinstance(result, dict):
+ for k, v in expected.items():
+ if result.get(k) != v:
+ raise AssertionError(
+ f"action={action!r} expected {k}={v!r}, got {result.get(k)!r} (full: {result!r})",
+ )
+ elif result != expected:
+ raise AssertionError(f"action={action!r} expected {expected!r}, got {result!r}")
+ return result
diff --git a/reme4/utils/env_utils.py b/reme4/utils/env_utils.py
new file mode 100644
index 00000000..7c0aecbc
--- /dev/null
+++ b/reme4/utils/env_utils.py
@@ -0,0 +1,36 @@
+"""Load .env files into os.environ (idempotent)."""
+
+import os
+from pathlib import Path
+
+_LOADED = False
+
+
+def _parse(path: Path) -> None:
+ for line in path.read_text(encoding="utf-8").splitlines():
+ line = line.strip()
+ if not line or line.startswith("#") or "=" not in line:
+ continue
+ key, value = line.split("=", 1)
+ os.environ[key.strip()] = value.strip().strip("'\"")
+
+
+def load_env(path: str | Path | None = None) -> None:
+ """Load .env from given path, or search cwd and up to 5 parents."""
+ global _LOADED
+ if _LOADED:
+ return
+
+ if path:
+ path = Path(path)
+ if path.exists():
+ _parse(path)
+ _LOADED = True
+ return
+
+ for directory in [Path.cwd(), *Path.cwd().parents[:5]]:
+ env_path = directory / ".env"
+ if env_path.exists():
+ _parse(env_path)
+ _LOADED = True
+ return
diff --git a/reme4/utils/logger_utils.py b/reme4/utils/logger_utils.py
new file mode 100644
index 00000000..53d63b67
--- /dev/null
+++ b/reme4/utils/logger_utils.py
@@ -0,0 +1,108 @@
+"""Logger utilities supporting both loguru and standard logging backends."""
+
+import logging
+import os
+import sys
+from datetime import datetime
+from logging.handlers import TimedRotatingFileHandler
+
+_logger = None
+
+_LOGURU_FORMAT = "{time:YYYY-MM-DD HH:mm:ss} | {level} | {file}:{line} | {function} | {message}"
+_STDLIB_FORMAT = "%(asctime)s | %(levelname)s | %(filename)s:%(lineno)d | %(funcName)s | %(message)s"
+_STDLIB_DATEFMT = "%Y-%m-%d %H:%M:%S"
+
+
+def _enable_loguru() -> bool:
+ return os.getenv("REME_DISABLE_LOGURU", "").lower() != "true"
+
+
+def _init_loguru(log_dir: str, level: str, log_to_console: bool, log_to_file: bool):
+ from loguru import logger
+
+ logger.remove()
+
+ if log_to_console:
+ logger.add(
+ sink=sys.stdout,
+ level=level,
+ format=_LOGURU_FORMAT,
+ colorize=True,
+ )
+
+ if log_to_file:
+ try:
+ os.makedirs(log_dir, exist_ok=True)
+ current_ts = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+ log_filepath = os.path.join(log_dir, f"{current_ts}.log")
+
+ logger.add(
+ log_filepath,
+ level=level,
+ rotation="00:00",
+ retention="7 days",
+ compression="zip",
+ encoding="utf-8",
+ format=_LOGURU_FORMAT,
+ )
+ except Exception as e:
+ logger.error(f"Error configuring file logging: {e}")
+
+ return logger
+
+
+def _init_stdlib(log_dir: str, level: str, log_to_console: bool, log_to_file: bool):
+ logger = logging.getLogger("reme")
+ logger.setLevel(level)
+ logger.propagate = False
+
+ for handler in list(logger.handlers):
+ logger.removeHandler(handler)
+
+ formatter = logging.Formatter(_STDLIB_FORMAT, datefmt=_STDLIB_DATEFMT)
+
+ if log_to_console:
+ console_handler = logging.StreamHandler(sys.stdout)
+ console_handler.setLevel(level)
+ console_handler.setFormatter(formatter)
+ logger.addHandler(console_handler)
+
+ if log_to_file:
+ try:
+ os.makedirs(log_dir, exist_ok=True)
+ current_ts = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+ log_filepath = os.path.join(log_dir, f"{current_ts}.log")
+
+ file_handler = TimedRotatingFileHandler(
+ log_filepath,
+ when="midnight",
+ backupCount=7,
+ encoding="utf-8",
+ )
+ file_handler.setLevel(level)
+ file_handler.setFormatter(formatter)
+ logger.addHandler(file_handler)
+ except Exception as e:
+ logger.error(f"Error configuring file logging: {e}")
+
+ return logger
+
+
+def get_logger(
+ log_dir: str = "logs",
+ level: str = "INFO",
+ log_to_console: bool = True,
+ log_to_file: bool = True,
+ force_init: bool = False,
+):
+ """Return the global logger, initializing sinks on first call (or when force_init)."""
+ global _logger
+
+ if _logger is not None and not force_init:
+ return _logger
+
+ if _enable_loguru():
+ _logger = _init_loguru(log_dir, level, log_to_console, log_to_file)
+ else:
+ _logger = _init_stdlib(log_dir, level, log_to_console, log_to_file)
+ return _logger
diff --git a/reme4/utils/logo_utils.py b/reme4/utils/logo_utils.py
new file mode 100644
index 00000000..2a70ab94
--- /dev/null
+++ b/reme4/utils/logo_utils.py
@@ -0,0 +1,87 @@
+"""Startup banner with ASCII logo and service metadata."""
+
+import importlib.metadata
+from typing import TYPE_CHECKING
+
+from rich.console import Console, Group
+from rich.panel import Panel
+from rich.table import Table
+from rich.text import Text
+
+if TYPE_CHECKING:
+ from ..schema import ApplicationConfig
+
+
+def get_version(package_name: str) -> str:
+ """Return installed package version, or empty string if not installed."""
+ try:
+ return importlib.metadata.version(package_name)
+ except importlib.metadata.PackageNotFoundError:
+ return ""
+
+
+def print_logo(app_config: "ApplicationConfig"):
+ """Print gradient ASCII logo and runtime config (backend, URL, versions)."""
+ ascii_art = [
+ r" ██████╗ ███████╗ ███╗ ███╗ ███████╗ ",
+ r" ██╔══██╗ ██╔════╝ ████╗ ████║ ██╔════╝ ",
+ r" ██████╔╝ █████╗ ██╔████╔██║ █████╗ ",
+ r" ██╔══██╗ ██╔══╝ ██║╚██╔╝██║ ██╔══╝ ",
+ r" ██║ ██║ ███████╗ ██║ ╚═╝ ██║ ███████╗ ",
+ r" ╚═╝ ╚═╝ ╚══════╝ ╚═╝ ╚═╝ ╚══════╝ ",
+ ]
+
+ start_color = (85, 239, 196)
+ end_color = (162, 155, 254)
+
+ logo_text = Text()
+ for line in ascii_art:
+ line_len = max(1, len(line) - 1)
+ for i, char in enumerate(line):
+ ratio = i / line_len
+ rgb = tuple(int(s + (e - s) * ratio) for s, e in zip(start_color, end_color))
+ logo_text.append(char, style=f"bold rgb({rgb[0]},{rgb[1]},{rgb[2]})")
+ logo_text.append("\n")
+
+ info_table = Table.grid(padding=(0, 1))
+ info_table.add_column(style="bold", justify="center")
+ info_table.add_column(style="bold cyan", justify="left")
+ info_table.add_column(style="white", justify="left")
+
+ # service is a ComponentConfig with extra="allow"; backend-specific fields live in model_extra.
+ service = app_config.service
+ backend = service.backend
+ extra = service.model_extra or {}
+
+ info_table.add_row("📦", "Backend:", backend)
+
+ match backend:
+ case "http":
+ host = extra.get("host", "localhost")
+ port = extra.get("port", 8000)
+ info_table.add_row("🔗", "URL:", f"http://{host}:{port}")
+ info_table.add_row("📚", "FastAPI:", Text(get_version("fastapi"), style="dim"))
+ case "mcp":
+ transport = extra.get("transport", "stdio")
+ info_table.add_row("🚌", "Transport:", transport)
+ if transport != "stdio":
+ host = extra.get("host", "localhost")
+ port = extra.get("port", 8000)
+ url = f"http://{host}:{port}"
+ if transport == "sse":
+ url += "/sse"
+ info_table.add_row("🔗", "URL:", url)
+ info_table.add_row("📚", "FastMCP:", Text(get_version("fastmcp"), style="dim"))
+
+ info_table.add_row("🚀", "ReMe:", Text(get_version("reme-ai"), style="dim"))
+
+ panel = Panel(
+ Group(logo_text, info_table),
+ title=app_config.app_name,
+ title_align="left",
+ border_style="dim",
+ padding=(1, 4),
+ expand=False,
+ )
+
+ Console().print(Group("\n", panel, "\n"))
diff --git a/reme4/utils/service_utils.py b/reme4/utils/service_utils.py
new file mode 100644
index 00000000..d1a39b9d
--- /dev/null
+++ b/reme4/utils/service_utils.py
@@ -0,0 +1,96 @@
+"""Service discovery utilities."""
+
+import asyncio
+import socket
+import subprocess
+import sys
+
+from ..constants import REME_DEFAULT_HOST, REME_DEFAULT_PORT
+
+
+async def find_reme(host: str, port: int) -> str:
+ """Probe host:port. Returns 'reme', 'occupied', or 'free'."""
+ from ..components.client.http_client import HttpClient
+
+ try:
+ async with HttpClient(action="health_check", host=host, port=port, timeout=2.0) as client:
+ async for _ in client():
+ break
+ return "reme"
+ except Exception:
+ pass
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+ s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ try:
+ s.bind((host, port))
+ return "free"
+ except OSError:
+ return "occupied"
+
+
+def _sh(cmd: list[str]) -> str:
+ """Run cmd; return stdout, or '' on failure."""
+ try:
+ return subprocess.check_output(cmd, stderr=subprocess.DEVNULL, text=True)
+ except (subprocess.CalledProcessError, FileNotFoundError):
+ return ""
+
+
+def _pid_on_port(port: int) -> int | None:
+ """PID listening on TCP port, or None."""
+ out = _sh(["lsof", "-nP", f"-iTCP:{port}", "-sTCP:LISTEN", "-t"]).strip()
+ return int(out.splitlines()[0]) if out else None
+
+
+def _scan_reme_procs() -> list[tuple[int, str, int]]:
+ """List running 'reme ... start' processes as (pid, host, port)."""
+ procs: list[tuple[int, str, int]] = []
+ for line in _sh(["pgrep", "-af", "reme.* start"]).splitlines():
+ parts = line.split()
+ if not parts or not parts[0].isdigit():
+ continue
+ host, port = REME_DEFAULT_HOST, REME_DEFAULT_PORT
+ for t in parts[1:]:
+ if t.startswith("service.host="):
+ host = t.split("=", 1)[1]
+ elif t.startswith("service.port=") and t.split("=", 1)[1].isdigit():
+ port = int(t.split("=", 1)[1])
+ procs.append((int(parts[0]), host, port))
+ return procs
+
+
+async def locate_reme() -> tuple[str, int, int | None] | None:
+ """Find a running reme: try default port, then scanned processes."""
+ if await find_reme(REME_DEFAULT_HOST, REME_DEFAULT_PORT) == "reme":
+ return REME_DEFAULT_HOST, REME_DEFAULT_PORT, _pid_on_port(REME_DEFAULT_PORT)
+ for pid, host, port in _scan_reme_procs():
+ if await find_reme(host, port) == "reme":
+ return host, port, pid
+ return None
+
+
+def precheck_start(svc_config: dict | None) -> bool:
+ """Pre-flight check for `start`: False if reme is up, exits 1 on port conflict."""
+ host = (svc_config or {}).get("host") or REME_DEFAULT_HOST
+ port = (svc_config or {}).get("port") or REME_DEFAULT_PORT
+ status = asyncio.run(find_reme(host, port))
+ if status == "reme":
+ print(f"reme already running at {host}:{port}")
+ return False
+ if status == "occupied":
+ print(
+ f"port {port} occupied. Start on another port: reme4 start service.port=",
+ file=sys.stderr,
+ )
+ sys.exit(1)
+ return True
+
+
+def cli_find_reme() -> None:
+ """Handle `reme find_reme`: print HOST/PORT/PID or a hint to start reme."""
+ found = asyncio.run(locate_reme())
+ if not found:
+ print("reme not started. Try: reme start", file=sys.stderr)
+ sys.exit(1)
+ host, port, pid = found
+ print(f"HOST={host} PORT={port} PID={pid or 'unknown'}")
diff --git a/reme4/utils/similarity_utils.py b/reme4/utils/similarity_utils.py
new file mode 100644
index 00000000..ec0f7100
--- /dev/null
+++ b/reme4/utils/similarity_utils.py
@@ -0,0 +1,35 @@
+"""Cosine similarity for single vectors and batched matrices."""
+
+import numpy as np
+
+
+def cosine_similarity(vec1: list[float], vec2: list[float]) -> float:
+ """Cosine similarity of two equal-length vectors; returns 0.0 if either has zero norm."""
+ if len(vec1) != len(vec2):
+ raise ValueError(f"Vectors must have same length: {len(vec1)} != {len(vec2)}")
+
+ dot_product = sum(a * b for a, b in zip(vec1, vec2))
+ magnitude1 = sum(a * a for a in vec1) ** 0.5
+ magnitude2 = sum(b * b for b in vec2) ** 0.5
+
+ if magnitude1 == 0 or magnitude2 == 0:
+ return 0.0
+
+ return dot_product / (magnitude1 * magnitude2)
+
+
+def batch_cosine_similarity(nd_array1: np.ndarray, nd_array2: np.ndarray) -> np.ndarray:
+ """Pairwise cosine similarity matrix between two batches; output shape (N1, N2)."""
+ if nd_array1.shape[1] != nd_array2.shape[1]:
+ raise ValueError(
+ f"Embedding dimensions must match: {nd_array1.shape[1]} != {nd_array2.shape[1]}",
+ )
+
+ dot_products = np.dot(nd_array1, nd_array2.T)
+ norms1 = np.linalg.norm(nd_array1, axis=1)
+ norms2 = np.linalg.norm(nd_array2, axis=1)
+ norm_products = np.outer(norms1, norms2)
+ # Guard against zero-norm rows to keep division finite.
+ norm_products = np.where(norm_products == 0, 1e-10, norm_products)
+
+ return dot_products / norm_products
diff --git a/tests4/unittest/test_bm25_index_perf.py b/tests4/unittest/test_bm25_index_perf.py
new file mode 100644
index 00000000..b6ba0f34
--- /dev/null
+++ b/tests4/unittest/test_bm25_index_perf.py
@@ -0,0 +1,337 @@
+"""BM25Index performance tests for add_docs and retrieve."""
+
+import asyncio
+import os
+import random
+import tempfile
+import time
+
+from reme4.components.keyword_index import BM25Index
+from reme4.components.tokenizer import RegexTokenizer
+
+# A small vocab of realistic-looking words for generating random text
+_VOCAB = [
+ "algorithm",
+ "data",
+ "machine",
+ "learning",
+ "model",
+ "network",
+ "neural",
+ "training",
+ "optimization",
+ "gradient",
+ "loss",
+ "function",
+ "parameter",
+ "weight",
+ "bias",
+ "layer",
+ "activation",
+ "relu",
+ "sigmoid",
+ "softmax",
+ "backpropagation",
+ "forward",
+ "pass",
+ "batch",
+ "epoch",
+ "iteration",
+ "convergence",
+ "divergence",
+ "regularization",
+ "dropout",
+ "attention",
+ "transformer",
+ "encoder",
+ "decoder",
+ "embedding",
+ "token",
+ "vector",
+ "matrix",
+ "tensor",
+ "computation",
+ "graph",
+ "node",
+ "edge",
+ "vertex",
+ "path",
+ "search",
+ "retrieval",
+ "index",
+ "query",
+ "document",
+ "corpus",
+ "term",
+ "frequency",
+ "inverse",
+ "score",
+ "rank",
+ "relevance",
+ "precision",
+ "recall",
+ "f1",
+ "metric",
+ "evaluation",
+ "benchmark",
+ "dataset",
+ "sample",
+ "feature",
+ "label",
+ "class",
+ "predict",
+ "classification",
+ "regression",
+ "clustering",
+ "dimension",
+ "reduction",
+ "pca",
+ "tsne",
+ "visualization",
+ "matplotlib",
+ "plot",
+ "chart",
+ "histogram",
+ "scatter",
+ "line",
+ "bar",
+ "database",
+ "sql",
+ "query",
+ "table",
+ "row",
+ "column",
+ "index",
+ "primary",
+ "foreign",
+ "key",
+ "constraint",
+ "schema",
+ "migration",
+ "version",
+ "control",
+ "git",
+ "commit",
+ "branch",
+ "merge",
+ "conflict",
+ "resolution",
+ "review",
+ "approve",
+ "reject",
+ "pull",
+ "request",
+ "issue",
+ "bug",
+ "fix",
+ "feature",
+ "enhancement",
+ "refactor",
+ "test",
+ "deploy",
+ "production",
+ "staging",
+ "development",
+ "environment",
+ "configuration",
+ "setting",
+ "variable",
+ "constant",
+ "global",
+ "local",
+ "scope",
+ "closure",
+ "callback",
+ "promise",
+ "async",
+ "await",
+ "synchronous",
+ "asynchronous",
+ "concurrent",
+ "parallel",
+ "thread",
+ "process",
+ "memory",
+ "cache",
+ "buffer",
+ "queue",
+ "stack",
+ "heap",
+ "pool",
+]
+
+
+class temp_chdir:
+ """Context manager to temporarily chdir into a path and restore on exit."""
+
+ def __init__(self, path):
+ self.path = path
+ self.old = None
+
+ def __enter__(self):
+ self.old = os.getcwd()
+ os.chdir(self.path)
+ return self
+
+ def __exit__(self, *exc):
+ os.chdir(self.old)
+
+
+def _gen_random_text(n_tokens: int) -> str:
+ """Generate random text with approximately n_tokens words."""
+ words = random.choices(_VOCAB, k=n_tokens)
+ return " ".join(words)
+
+
+def _gen_random_query(n_words: int) -> str:
+ """Generate a random query with n_words words."""
+ words = random.choices(_VOCAB, k=n_words)
+ return " ".join(words)
+
+
+async def _make_index() -> BM25Index:
+ """Create and start a BM25Index using cwd as working dir, with non-filtering tokenizer."""
+ index = BM25Index()
+ tokenizer = RegexTokenizer(filter_stopwords=False)
+ index.tokenizer = tokenizer
+ index._owned.append(tokenizer) # pylint: disable=protected-access
+ await index.start()
+ return index
+
+
+async def _setup_index_for_retrieve(n_docs: int = 100, doc_tokens: int = 1000) -> BM25Index:
+ """Build an index with n_docs medium-sized docs in cwd."""
+ index = await _make_index()
+ docs = {f"doc_{i}": _gen_random_text(doc_tokens) for i in range(n_docs)}
+ await index.add_docs(docs)
+ return index
+
+
+def test_add_docs_small():
+ """Add 100 small docs (~100 tokens each)."""
+
+ async def run():
+ docs = {f"doc_{i}": _gen_random_text(100) for i in range(100)}
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ index = await _make_index()
+ t0 = time.perf_counter()
+ await index.add_docs(docs)
+ elapsed = time.perf_counter() - t0
+ print(f" add_docs (100 docs x ~100 tokens): {elapsed:.4f}s")
+ await index.close()
+
+ asyncio.run(run())
+
+
+def test_add_docs_medium():
+ """Add 100 medium docs (~1000 tokens each)."""
+
+ async def run():
+ docs = {f"doc_{i}": _gen_random_text(1000) for i in range(100)}
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ index = await _make_index()
+ t0 = time.perf_counter()
+ await index.add_docs(docs)
+ elapsed = time.perf_counter() - t0
+ print(f" add_docs (100 docs x ~1000 tokens): {elapsed:.4f}s")
+ await index.close()
+
+ asyncio.run(run())
+
+
+def test_add_docs_large():
+ """Add 100 large docs (~10000 tokens each)."""
+
+ async def run():
+ docs = {f"doc_{i}": _gen_random_text(10000) for i in range(100)}
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ index = await _make_index()
+ t0 = time.perf_counter()
+ await index.add_docs(docs)
+ elapsed = time.perf_counter() - t0
+ print(f" add_docs (100 docs x ~10000 tokens): {elapsed:.4f}s")
+ await index.close()
+
+ asyncio.run(run())
+
+
+def test_retrieve_short_query():
+ """Retrieve with 1-word query."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ index = await _setup_index_for_retrieve()
+ query = _gen_random_query(1)
+ t0 = time.perf_counter()
+ await index.retrieve(query, limit=10)
+ elapsed = time.perf_counter() - t0
+ print(f" retrieve (1-word query, 100 docs): {elapsed:.6f}s")
+ await index.close()
+
+ asyncio.run(run())
+
+
+def test_retrieve_medium_query():
+ """Retrieve with 5-word query."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ index = await _setup_index_for_retrieve()
+ query = _gen_random_query(5)
+ t0 = time.perf_counter()
+ await index.retrieve(query, limit=10)
+ elapsed = time.perf_counter() - t0
+ print(f" retrieve (5-word query, 100 docs): {elapsed:.6f}s")
+ await index.close()
+
+ asyncio.run(run())
+
+
+def test_retrieve_long_query():
+ """Retrieve with 20-word query."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ index = await _setup_index_for_retrieve()
+ query = _gen_random_query(20)
+ t0 = time.perf_counter()
+ await index.retrieve(query, limit=10)
+ elapsed = time.perf_counter() - t0
+ print(f" retrieve (20-word query, 100 docs): {elapsed:.6f}s")
+ await index.close()
+
+ asyncio.run(run())
+
+
+def test_retrieve_very_long_query():
+ """Retrieve with 100-word query."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ index = await _setup_index_for_retrieve()
+ query = _gen_random_query(100)
+ t0 = time.perf_counter()
+ await index.retrieve(query, limit=10)
+ elapsed = time.perf_counter() - t0
+ print(f" retrieve (100-word query, 100 docs): {elapsed:.6f}s")
+ await index.close()
+
+ asyncio.run(run())
+
+
+if __name__ == "__main__":
+ random.seed(42)
+ print("=== BM25Index Performance Tests ===\n")
+
+ print("[add_docs]")
+ test_add_docs_small()
+ test_add_docs_medium()
+ test_add_docs_large()
+
+ print("\n[retrieve]")
+ test_retrieve_short_query()
+ test_retrieve_medium_query()
+ test_retrieve_long_query()
+ test_retrieve_very_long_query()
+
+ print("\nDone.")
diff --git a/tests4/unittest/test_bm25_lite.py b/tests4/unittest/test_bm25_lite.py
new file mode 100644
index 00000000..75865797
--- /dev/null
+++ b/tests4/unittest/test_bm25_lite.py
@@ -0,0 +1,586 @@
+"""Tests for BM25Index search engine."""
+
+# pylint: disable=protected-access
+
+import asyncio
+import os
+import tempfile
+import warnings
+
+from reme4.components.keyword_index import BM25Index
+from reme4.components.tokenizer import RegexTokenizer
+
+# Filter jieba/pkg_resources deprecation warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba")
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources")
+
+
+class temp_chdir:
+ """Context manager to temporarily chdir into a path and restore on exit."""
+
+ def __init__(self, path):
+ self.path = path
+ self.old = None
+
+ def __enter__(self):
+ self.old = os.getcwd()
+ os.chdir(self.path)
+ return self
+
+ def __exit__(self, *exc):
+ os.chdir(self.old)
+
+
+async def create_bm25(k1: float = 1.5, b: float = 0.75) -> BM25Index:
+ """Create and start a BM25Index in cwd with a non-filtering tokenizer.
+
+ The non-filtering tokenizer keeps short test texts (e.g. "hello world") visible,
+ since several common test words ("hello", "我", "的") are in the default stopwords.
+ """
+ bm25 = BM25Index(k1=k1, b=b)
+ # Replace the unresolved Dependency placeholder with a real tokenizer instance.
+ tokenizer = RegexTokenizer(filter_stopwords=False)
+ bm25.tokenizer = tokenizer
+ bm25._owned.append(tokenizer)
+ await bm25.start()
+ return bm25
+
+
+def test_basic_init():
+ """Test BM25Index initialization."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = BM25Index()
+ assert bm25.k1 == 1.5
+ assert bm25.b == 0.75
+ assert bm25.vocab == {}
+ assert bm25.inverted_index == {}
+ assert bm25.doc_meta == {}
+ assert bm25.n_docs == 0
+ assert bm25.avg_len == 0.0
+ print("✓ test_basic_init passed")
+
+ asyncio.run(run())
+
+
+def test_start_with_tokenizer():
+ """Test BM25Index starts and initializes tokenizer."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+ assert bm25.tokenizer is not None
+ assert bm25.is_started
+
+ await bm25.close()
+ assert not bm25.is_started
+ print("✓ test_start_with_tokenizer passed")
+
+ asyncio.run(run())
+
+
+def test_add_single_doc():
+ """Test adding a single document."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ await bm25.add_docs({"doc1": "hello world"})
+
+ assert bm25.n_docs == 1
+ assert bm25.total_len > 0
+ assert "doc1" in bm25.doc_meta
+
+ await bm25.close()
+ print("✓ test_add_single_doc passed")
+
+ asyncio.run(run())
+
+
+def test_add_multiple_docs():
+ """Test adding multiple documents."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ docs = {
+ "doc1": "hello world",
+ "doc2": "hello python",
+ "doc3": "world python",
+ }
+ await bm25.add_docs(docs)
+
+ assert bm25.n_docs == 3
+ assert len(bm25.vocab) > 0
+ assert len(bm25.inverted_index) > 0
+
+ await bm25.close()
+ print("✓ test_add_multiple_docs passed")
+
+ asyncio.run(run())
+
+
+def test_retrieve_basic():
+ """Test basic retrieval functionality."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ docs = {
+ "doc1": "python programming language",
+ "doc2": "java programming language",
+ "doc3": "python data analysis",
+ }
+ await bm25.add_docs(docs)
+
+ results = await bm25.retrieve("python", limit=3)
+ assert len(results) <= 3
+ assert "doc1" in results or "doc3" in results
+
+ await bm25.close()
+ print("✓ test_retrieve_basic passed")
+
+ asyncio.run(run())
+
+
+def test_retrieve_with_limit():
+ """Test retrieval with result limit."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ docs = {f"doc{i}": f"python programming {i}" for i in range(10)}
+ await bm25.add_docs(docs)
+
+ results = await bm25.retrieve("python", limit=3)
+ assert len(results) == 3
+
+ results = await bm25.retrieve("python", limit=5)
+ assert len(results) == 5
+
+ await bm25.close()
+ print("✓ test_retrieve_with_limit passed")
+
+ asyncio.run(run())
+
+
+def test_retrieve_empty_query():
+ """Test retrieval with empty or unknown query."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ docs = {"doc1": "hello world"}
+ await bm25.add_docs(docs)
+
+ results = await bm25.retrieve("", limit=3)
+ assert results == {}
+
+ results = await bm25.retrieve("unknownxyz", limit=3)
+ assert results == {}
+
+ await bm25.close()
+ print("✓ test_retrieve_empty_query passed")
+
+ asyncio.run(run())
+
+
+def test_retrieve_empty_index():
+ """Test retrieval from empty index."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ results = await bm25.retrieve("python", limit=3)
+ assert results == {}
+
+ await bm25.close()
+ print("✓ test_retrieve_empty_index passed")
+
+ asyncio.run(run())
+
+
+def test_update_doc():
+ """Test updating an existing document."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ await bm25.add_docs({"doc1": "hello world python"})
+ old_len = bm25.total_len
+
+ await bm25.add_docs({"doc1": "java"})
+ assert bm25.n_docs == 1
+ assert bm25.total_len != old_len
+
+ results = await bm25.retrieve("java", limit=1)
+ assert "doc1" in results
+
+ await bm25.close()
+ print("✓ test_update_doc passed")
+
+ asyncio.run(run())
+
+
+def test_remove_doc():
+ """Test removing a document."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ docs = {
+ "doc1": "hello world",
+ "doc2": "hello python",
+ }
+ await bm25.add_docs(docs)
+ assert bm25.n_docs == 2
+
+ bm25._remove_doc("doc1")
+ assert bm25.n_docs == 1
+ assert "doc1" not in bm25.doc_meta
+
+ results = await bm25.retrieve("hello", limit=2)
+ assert "doc1" not in results
+ assert "doc2" in results
+
+ await bm25.close()
+ print("✓ test_remove_doc passed")
+
+ asyncio.run(run())
+
+
+def test_remove_nonexistent_doc():
+ """Test removing a nonexistent document (should be no-op)."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ await bm25.add_docs({"doc1": "hello world"})
+ bm25._remove_doc("nonexistent")
+ assert bm25.n_docs == 1
+
+ await bm25.close()
+ print("✓ test_remove_nonexistent_doc passed")
+
+ asyncio.run(run())
+
+
+def test_clear():
+ """Test clearing the index."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ await bm25.add_docs(
+ {
+ "doc1": "hello world",
+ "doc2": "hello python",
+ },
+ )
+ assert bm25.n_docs == 2
+
+ await bm25.clear()
+ assert bm25.n_docs == 0
+ assert bm25.vocab == {}
+ assert bm25.inverted_index == {}
+ assert bm25.doc_meta == {}
+ assert bm25.total_len == 0
+ assert bm25._idf_cache == {}
+
+ await bm25.close()
+ print("✓ test_clear passed")
+
+ asyncio.run(run())
+
+
+def test_optimize_index():
+ """Test optimize_index functionality to compact vocab."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ await bm25.add_docs({"doc1": "hello world"})
+ bm25._remove_doc("doc1")
+
+ assert bm25.n_docs == 0
+ assert len(bm25.vocab) > 0
+
+ await bm25.optimize_index()
+ assert bm25.vocab == {}
+ assert bm25.inverted_index == {}
+
+ await bm25.close()
+ print("✓ test_optimize_index passed")
+
+ asyncio.run(run())
+
+
+def test_optimize_index_with_docs():
+ """Test optimize_index with remaining documents."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ await bm25.add_docs(
+ {
+ "doc1": "hello world",
+ "doc2": "hello python",
+ },
+ )
+
+ old_vocab = bm25.vocab.copy()
+ bm25._remove_doc("doc1")
+
+ await bm25.optimize_index()
+
+ assert bm25.n_docs == 1
+ assert "doc2" in bm25.doc_meta
+ assert len(bm25.vocab) < len(old_vocab)
+
+ results = await bm25.retrieve("hello", limit=1)
+ assert "doc2" in results
+
+ await bm25.close()
+ print("✓ test_optimize_index_with_docs passed")
+
+ asyncio.run(run())
+
+
+def test_persistence():
+ """Test dump and load persistence."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+ docs = {
+ "doc1": "hello world",
+ "doc2": "hello python",
+ "doc3": "programming language",
+ }
+ await bm25.add_docs(docs)
+
+ old_vocab = bm25.vocab.copy()
+ old_doc_meta = {k: dict(v) for k, v in bm25.doc_meta.items()}
+
+ await bm25.dump()
+ await bm25.close()
+
+ bm25_new = await create_bm25()
+
+ assert bm25_new.vocab == old_vocab
+ assert bm25_new.n_docs == 3
+ for doc_id in old_doc_meta:
+ assert doc_id in bm25_new.doc_meta
+
+ results = await bm25_new.retrieve("hello", limit=2)
+ assert "doc1" in results or "doc2" in results
+
+ await bm25_new.close()
+ print("✓ test_persistence passed")
+
+ asyncio.run(run())
+
+
+def test_custom_params():
+ """Test custom k1 and b parameters."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25(k1=2.0, b=0.5)
+
+ assert bm25.k1 == 2.0
+ assert bm25.b == 0.5
+
+ await bm25.add_docs({"doc1": "test document"})
+ results = await bm25.retrieve("test", limit=1)
+ assert "doc1" in results
+
+ await bm25.close()
+ print("✓ test_custom_params passed")
+
+ asyncio.run(run())
+
+
+def test_chinese_text():
+ """Test with Chinese text."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ docs = {
+ "doc1": "我爱北京天安门",
+ "doc2": "北京是中国的首都",
+ "doc3": "上海的天气很好",
+ }
+ await bm25.add_docs(docs)
+
+ results = await bm25.retrieve("北", limit=2)
+ assert len(results) <= 2
+ assert "doc1" in results or "doc2" in results
+
+ await bm25.close()
+ print("✓ test_chinese_text passed")
+
+ asyncio.run(run())
+
+
+def test_mixed_chinese_english():
+ """Test with mixed Chinese and English text."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ docs = {
+ "doc1": "Python 是一种编程语言",
+ "doc2": "Java 编程语言",
+ "doc3": "Python 数据分析",
+ }
+ await bm25.add_docs(docs)
+
+ results = await bm25.retrieve("Python", limit=3)
+ assert len(results) > 0
+
+ results = await bm25.retrieve("编", limit=2)
+ assert len(results) > 0
+
+ await bm25.close()
+ print("✓ test_mixed_chinese_english passed")
+
+ asyncio.run(run())
+
+
+def test_idf_cache():
+ """Test IDF cache functionality."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ await bm25.add_docs(
+ {
+ "doc1": "hello world",
+ "doc2": "hello python",
+ },
+ )
+
+ token = "hello"
+ if token in bm25.vocab:
+ tid = bm25.vocab[token]
+ idf1 = bm25._get_idf(tid)
+ assert tid in bm25._idf_cache
+ idf2 = bm25._get_idf(tid)
+ assert idf1 == idf2
+
+ await bm25.close()
+ print("✓ test_idf_cache passed")
+
+ asyncio.run(run())
+
+
+def test_avg_len():
+ """Test average document length calculation."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ assert bm25.avg_len == 0.0
+
+ await bm25.add_docs({"doc1": "hello world python"})
+ assert bm25.avg_len > 0
+
+ await bm25.add_docs({"doc2": "test"})
+ new_avg = bm25.avg_len
+ assert new_avg > 0
+
+ await bm25.close()
+ print("✓ test_avg_len passed")
+
+ asyncio.run(run())
+
+
+def test_score_ordering():
+ """Test that results are ordered by score descending."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ docs = {
+ "doc1": "python python python",
+ "doc2": "python python",
+ "doc3": "python",
+ }
+ await bm25.add_docs(docs)
+
+ results = await bm25.retrieve("python", limit=3)
+ scores = list(results.values())
+
+ for i in range(len(scores) - 1):
+ assert scores[i] >= scores[i + 1]
+
+ await bm25.close()
+ print("✓ test_score_ordering passed")
+
+ asyncio.run(run())
+
+
+def test_empty_doc():
+ """Test adding empty document."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ bm25 = await create_bm25()
+
+ await bm25.add_docs({"doc1": ""})
+ assert bm25.n_docs == 0
+
+ await bm25.add_docs({"doc2": " "})
+ assert bm25.n_docs == 0
+
+ await bm25.close()
+ print("✓ test_empty_doc passed")
+
+ asyncio.run(run())
+
+
+if __name__ == "__main__":
+ print("\n=== BM25Index Tests ===")
+ test_basic_init()
+ test_start_with_tokenizer()
+ test_add_single_doc()
+ test_add_multiple_docs()
+ test_retrieve_basic()
+ test_retrieve_with_limit()
+ test_retrieve_empty_query()
+ test_retrieve_empty_index()
+ test_update_doc()
+ test_remove_doc()
+ test_remove_nonexistent_doc()
+ test_clear()
+ test_optimize_index()
+ test_optimize_index_with_docs()
+ test_persistence()
+ test_custom_params()
+ test_chinese_text()
+ test_mixed_chinese_english()
+ test_idf_cache()
+ test_avg_len()
+ test_score_ordering()
+ test_empty_doc()
+ print("\n所有测试通过!")
diff --git a/tests4/unittest/test_common_steps.py b/tests4/unittest/test_common_steps.py
new file mode 100644
index 00000000..ad7d5f5a
--- /dev/null
+++ b/tests4/unittest/test_common_steps.py
@@ -0,0 +1,290 @@
+"""End-to-end tests for reme4 common steps: spawn `reme4 start`, drive via HTTP,
+verify responses, then shut down. Each test uses an isolated cwd so the working_dir
+(.reme by default) does not collide.
+"""
+
+import asyncio
+import os
+import tempfile
+import warnings
+
+from reme4 import __version__ as REME_VERSION
+from reme4.utils import call_action, call_and_check, mock_reme_server
+
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba")
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources")
+
+
+class _temp_chdir:
+ """chdir to path for the duration of the block; restore on exit."""
+
+ def __init__(self, path):
+ self.path = path
+ self._old = None
+
+ def __enter__(self):
+ self._old = os.getcwd()
+ os.chdir(self.path)
+ return self
+
+ def __exit__(self, *exc):
+ os.chdir(self._old)
+
+
+def _run(coro):
+ """Run an async coroutine on a fresh isolated event loop."""
+ asyncio.run(coro)
+
+
+# ---------------------------------------------------------------------------
+# Individual job tests
+# ---------------------------------------------------------------------------
+
+
+def test_version_job():
+ """version job should return the package version string."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ async with mock_reme_server() as (host, port):
+ await call_and_check(
+ "version",
+ host=host,
+ port=port,
+ validator=lambda r: (
+ isinstance(r, dict)
+ and r.get("success") is True
+ and r.get("answer") == REME_VERSION
+ and r.get("metadata", {}).get("version") == REME_VERSION
+ ),
+ )
+ print("✓ test_version_job passed")
+
+ _run(run())
+
+
+def test_help_job():
+ """help job should list jobs except itself."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ async with mock_reme_server() as (host, port):
+ result = await call_and_check(
+ "help",
+ host=host,
+ port=port,
+ validator=lambda r: (
+ isinstance(r, dict)
+ and r.get("success") is True
+ and isinstance(r.get("answer"), str)
+ and r.get("metadata", {}).get("job_count", 0) > 0
+ and "help" not in r["answer"]
+ ),
+ )
+ # Spot-check that a couple of known jobs appear in the listing.
+ answer = result["answer"]
+ for expected_job in ("version", "health_check", "search"):
+ if expected_job not in answer:
+ raise AssertionError(f"help output missing job {expected_job!r}: {answer!r}")
+ print("✓ test_help_job passed")
+
+ _run(run())
+
+
+def test_health_check_job():
+ """health_check job should return a structured health snapshot."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ async with mock_reme_server() as (host, port):
+ result = await call_and_check(
+ "health_check",
+ host=host,
+ port=port,
+ validator=lambda r: (
+ isinstance(r, dict)
+ and r.get("success") is True
+ and isinstance(r.get("metadata"), dict)
+ and isinstance(r["metadata"].get("health"), dict)
+ and r["metadata"]["health"].get("version") == REME_VERSION
+ and isinstance(r["metadata"]["health"].get("components"), dict)
+ ),
+ )
+ # Validate that each expected component type is in the snapshot.
+ components = result["metadata"]["health"]["components"]
+ for ctype in (
+ "embedding_model",
+ "file_graph",
+ "file_store",
+ "file_watcher",
+ "keyword_index",
+ ):
+ if ctype not in components:
+ raise AssertionError(f"health snapshot missing component {ctype!r}: {components!r}")
+ print("✓ test_health_check_job passed")
+
+ _run(run())
+
+
+def test_search_job_empty_store():
+ """search on an empty store should return successfully with zero results."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ async with mock_reme_server() as (host, port):
+ await call_and_check(
+ "search",
+ host=host,
+ port=port,
+ query="hello world",
+ limit=5,
+ validator=lambda r: (
+ isinstance(r, dict)
+ and r.get("success") is True
+ and isinstance(r.get("metadata"), dict)
+ and isinstance(r["metadata"].get("counts"), dict)
+ and r["metadata"]["counts"].get("returned", -1) == 0
+ ),
+ )
+ print("✓ test_search_job_empty_store passed")
+
+ _run(run())
+
+
+def test_search_job_missing_query():
+ """search without a query should surface the assertion error in `answer`."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ async with mock_reme_server() as (host, port):
+ result = await call_action("search", host=host, port=port, query="")
+ if not isinstance(result, dict):
+ raise AssertionError(f"expected dict response, got {result!r}")
+ if "query" not in str(result.get("answer", "")).lower():
+ raise AssertionError(f"expected query-related error in answer, got {result!r}")
+ print("✓ test_search_job_missing_query passed")
+
+ _run(run())
+
+
+def test_reindex_job():
+ """reindex job should wipe the file store and rebuild from tracked files."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ async with mock_reme_server() as (host, port):
+ await call_and_check(
+ "reindex",
+ host=host,
+ port=port,
+ validator=lambda r: (
+ isinstance(r, dict)
+ and r.get("success") is True
+ and isinstance(r.get("metadata", {}).get("counts"), dict)
+ and "added" in r["metadata"]["counts"]
+ ),
+ )
+ print("✓ test_reindex_job passed")
+
+ _run(run())
+
+
+def test_demo_job():
+ """demo job should echo back the normalized query and adjusted min_score."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ async with mock_reme_server() as (host, port):
+ await call_and_check(
+ "demo",
+ host=host,
+ port=port,
+ query=" Hello World ",
+ min_score=0.8,
+ validator=lambda r: (
+ isinstance(r, dict)
+ and r.get("success") is True
+ and "hello world" in str(r.get("answer", ""))
+ and abs(r.get("metadata", {}).get("adjusted_min_score", 0) - 0.72) < 1e-6
+ ),
+ )
+ print("✓ test_demo_job passed")
+
+ _run(run())
+
+
+# ---------------------------------------------------------------------------
+# Aggregate test: reuse one server instance for all jobs (faster).
+# ---------------------------------------------------------------------------
+
+
+def test_all_jobs_one_server():
+ """Run every common job against a single shared server for efficiency."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ async with mock_reme_server() as (host, port):
+ # version
+ await call_and_check(
+ "version",
+ host=host,
+ port=port,
+ validator=lambda r: isinstance(r, dict) and r.get("answer") == REME_VERSION,
+ )
+ # help
+ await call_and_check(
+ "help",
+ host=host,
+ port=port,
+ validator=lambda r: isinstance(r, dict) and r.get("metadata", {}).get("job_count", 0) > 0,
+ )
+ # health_check
+ await call_and_check(
+ "health_check",
+ host=host,
+ port=port,
+ validator=lambda r: isinstance(r, dict)
+ and isinstance(
+ r.get("metadata", {}).get("health"),
+ dict,
+ ),
+ )
+ # search (empty store)
+ await call_and_check(
+ "search",
+ host=host,
+ port=port,
+ query="anything",
+ validator=lambda r: isinstance(r, dict) and r.get("success") is True,
+ )
+ # reindex
+ await call_and_check(
+ "reindex",
+ host=host,
+ port=port,
+ validator=lambda r: isinstance(r, dict) and isinstance(r.get("metadata", {}).get("counts"), dict),
+ )
+ # demo
+ await call_and_check(
+ "demo",
+ host=host,
+ port=port,
+ query="Foo",
+ validator=lambda r: isinstance(r, dict) and "foo" in str(r.get("answer", "")),
+ )
+ print("✓ test_all_jobs_one_server passed")
+
+ _run(run())
+
+
+if __name__ == "__main__":
+ print("\n=== reme4 common steps E2E tests ===")
+ test_version_job()
+ test_help_job()
+ test_health_check_job()
+ test_search_job_empty_store()
+ test_search_job_missing_query()
+ test_reindex_job()
+ test_demo_job()
+ test_all_jobs_one_server()
+ print("\n所有测试通过!")
diff --git a/tests4/unittest/test_default_file_parser.py b/tests4/unittest/test_default_file_parser.py
new file mode 100644
index 00000000..5730078b
--- /dev/null
+++ b/tests4/unittest/test_default_file_parser.py
@@ -0,0 +1,387 @@
+"""Tests for DefaultFileParser."""
+
+import asyncio
+import os
+import tempfile
+
+from reme4.components.file_parser import DefaultFileParser
+
+
+# Add parent path for import
+
+
+def test_parse_empty_file():
+ """Test parsing an empty file."""
+
+ async def run():
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f:
+ temp_path = f.name
+
+ try:
+ parser = DefaultFileParser()
+ file_node, chunks = await parser.parse(temp_path)
+ assert file_node.path == temp_path
+ assert len(chunks) == 0
+ print("✓ test_parse_empty_file passed")
+ finally:
+ os.unlink(temp_path)
+
+ asyncio.run(run())
+
+
+def test_parse_small_file():
+ """Test parsing a file smaller than chunk size."""
+
+ async def run():
+ content = "Hello World\nThis is a test\nLine 3"
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f:
+ f.write(content)
+ temp_path = f.name
+
+ try:
+ parser = DefaultFileParser(chunk_byte_size=10000)
+ _, chunks = await parser.parse(temp_path)
+ assert len(chunks) == 1
+ assert chunks[0].start_line == 1
+ assert chunks[0].end_line == 3
+ assert chunks[0].text == content
+ print("✓ test_parse_small_file passed")
+ finally:
+ os.unlink(temp_path)
+
+ asyncio.run(run())
+
+
+def test_parse_multiline_file():
+ """Test parsing a file with multiple lines."""
+
+ async def run():
+ lines = ["Line 1", "Line 2", "Line 3", "Line 4", "Line 5"]
+ content = "\n".join(lines)
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f:
+ f.write(content)
+ temp_path = f.name
+
+ try:
+ parser = DefaultFileParser(chunk_byte_size=10000)
+ _, chunks = await parser.parse(temp_path)
+ assert len(chunks) == 1
+ assert chunks[0].start_line == 1
+ assert chunks[0].end_line == 5
+ print("✓ test_parse_multiline_file passed")
+ finally:
+ os.unlink(temp_path)
+
+ asyncio.run(run())
+
+
+def test_parse_chunked_file():
+ """Test parsing a file that requires multiple chunks."""
+
+ async def run():
+ # Create content larger than chunk size
+ lines = ["A" * 100 for _ in range(200)] # ~20200 bytes
+ content = "\n".join(lines)
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f:
+ f.write(content)
+ temp_path = f.name
+
+ try:
+ parser = DefaultFileParser(chunk_byte_size=5000, overlap_byte_size=100)
+ _, chunks = await parser.parse(temp_path)
+ assert len(chunks) > 1, f"Expected multiple chunks, got {len(chunks)}"
+ # Verify overlap by checking that consecutive chunks share some content
+ print(f" Created {len(chunks)} chunks")
+ print("✓ test_parse_chunked_file passed")
+ finally:
+ os.unlink(temp_path)
+
+ asyncio.run(run())
+
+
+def test_parse_with_custom_encoding():
+ """Test parsing a file with different encodings."""
+
+ async def run():
+ content = "你好世界\n测试内容"
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt", encoding="utf-8") as f:
+ f.write(content)
+ temp_path = f.name
+
+ try:
+ parser = DefaultFileParser(encoding="utf-8")
+ _, chunks = await parser.parse(temp_path)
+ assert len(chunks) >= 1
+ assert "你好世界" in chunks[0].text
+ print("✓ test_parse_with_custom_encoding passed")
+ finally:
+ os.unlink(temp_path)
+
+ asyncio.run(run())
+
+
+def test_file_node_properties():
+ """Test FileNode has correct properties."""
+
+ async def run():
+ content = "test content"
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f:
+ f.write(content)
+ temp_path = f.name
+
+ try:
+ parser = DefaultFileParser()
+ file_node, _ = await parser.parse(temp_path)
+ assert hasattr(file_node, "path")
+ assert hasattr(file_node, "st_mtime")
+ assert file_node.st_mtime > 0
+ print("✓ test_file_node_properties passed")
+ finally:
+ os.unlink(temp_path)
+
+ asyncio.run(run())
+
+
+def test_file_chunk_properties():
+ """Test FileChunk has correct properties."""
+
+ async def run():
+ content = "test content for chunk"
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f:
+ f.write(content)
+ temp_path = f.name
+
+ try:
+ parser = DefaultFileParser()
+ _, chunks = await parser.parse(temp_path)
+ chunk = chunks[0]
+ assert hasattr(chunk, "path")
+ assert hasattr(chunk, "start_line")
+ assert hasattr(chunk, "end_line")
+ assert hasattr(chunk, "text")
+ assert hasattr(chunk, "id")
+ assert chunk.start_line >= 1
+ assert chunk.end_line >= chunk.start_line
+ print("✓ test_file_chunk_properties passed")
+ finally:
+ os.unlink(temp_path)
+
+ asyncio.run(run())
+
+
+def test_parse_links_bare():
+ """Bare wikilink: [[target]]."""
+ links = DefaultFileParser.parse_links("see [[note]]", "src.md")
+ assert len(links) == 1
+ link = links[0]
+ assert link.source_path == "src.md"
+ assert link.target_path == "note"
+ assert link.target_anchor is None
+ assert link.predicate is None
+ print("✓ test_parse_links_bare passed")
+
+
+def test_parse_links_with_anchor():
+ """Wikilink with anchor: [[target#anchor]]."""
+ links = DefaultFileParser.parse_links("see [[note#section A]]", "src.md")
+ assert len(links) == 1
+ assert links[0].target_path == "note"
+ assert links[0].target_anchor == "section A"
+ assert links[0].predicate is None
+ print("✓ test_parse_links_with_anchor passed")
+
+
+def test_parse_links_alias_dropped():
+ """Alias after '|' is consumed but not captured as anchor."""
+ links = DefaultFileParser.parse_links("see [[note|display text]]", "src.md")
+ assert len(links) == 1
+ assert links[0].target_path == "note"
+ assert links[0].target_anchor is None
+ print("✓ test_parse_links_alias_dropped passed")
+
+
+def test_parse_links_anchor_and_alias():
+ """[[target#anchor|alias]] — anchor captured, alias dropped."""
+ links = DefaultFileParser.parse_links("see [[note#sec|disp]]", "src.md")
+ assert len(links) == 1
+ assert links[0].target_path == "note"
+ assert links[0].target_anchor == "sec"
+ print("✓ test_parse_links_anchor_and_alias passed")
+
+
+def test_parse_links_predicate_simple():
+ """Dataview inline: predicate:: [[target]]."""
+ links = DefaultFileParser.parse_links("author:: [[Alice]]", "src.md")
+ assert len(links) == 1
+ assert links[0].predicate == "author"
+ assert links[0].target_path == "Alice"
+ assert links[0].target_anchor is None
+ print("✓ test_parse_links_predicate_simple passed")
+
+
+def test_parse_links_predicate_bracketed():
+ """Dataview inline-bracket: [predicate:: [[target]]]."""
+ links = DefaultFileParser.parse_links("text [author:: [[Alice]]] more", "src.md")
+ assert len(links) == 1
+ assert links[0].predicate == "author"
+ assert links[0].target_path == "Alice"
+ print("✓ test_parse_links_predicate_bracketed passed")
+
+
+def test_parse_links_predicate_bracketed_with_anchor():
+ """[predicate:: [[target_path#target_anchor]]] — combined form."""
+ links = DefaultFileParser.parse_links(
+ "[predicate:: [[target_path#target_anchor]]]",
+ "src.md",
+ )
+ assert len(links) == 1
+ link = links[0]
+ assert link.source_path == "src.md"
+ assert link.predicate == "predicate"
+ assert link.target_path == "target_path"
+ assert link.target_anchor == "target_anchor"
+ print("✓ test_parse_links_predicate_bracketed_with_anchor passed")
+
+
+def test_parse_links_predicate_sticks_to_first():
+ """Predicate attaches only to the immediately following wikilink."""
+ links = DefaultFileParser.parse_links("pred:: [[a]] and bare [[b]]", "src.md")
+ assert len(links) == 2
+ assert links[0].predicate == "pred" and links[0].target_path == "a"
+ assert links[1].predicate is None and links[1].target_path == "b"
+ print("✓ test_parse_links_predicate_sticks_to_first passed")
+
+
+def test_parse_links_multiple_on_one_line():
+ """Multiple bare wikilinks on the same line are all captured."""
+ links = DefaultFileParser.parse_links("see [[x]] and [[y#h]]", "src.md")
+ assert [(link.target_path, link.target_anchor) for link in links] == [
+ ("x", None),
+ ("y", "h"),
+ ]
+ print("✓ test_parse_links_multiple_on_one_line passed")
+
+
+def test_parse_links_no_match():
+ """Strings without [[]] yield no links, even if '::' appears."""
+ assert len(DefaultFileParser.parse_links("no link here :: foo", "src.md")) == 0
+ assert len(DefaultFileParser.parse_links("plain text without brackets", "src.md")) == 0
+ assert len(DefaultFileParser.parse_links("", "src.md")) == 0
+ print("✓ test_parse_links_no_match passed")
+
+
+def test_parse_links_predicate_with_dash_and_digits():
+ """Predicate identifier accepts letters, digits, underscore, dash."""
+ links = DefaultFileParser.parse_links("see-also-2:: [[target]]", "src.md")
+ assert len(links) == 1
+ assert links[0].predicate == "see-also-2"
+ assert links[0].target_path == "target"
+ print("✓ test_parse_links_predicate_with_dash_and_digits passed")
+
+
+def test_parse_links_in_file():
+ """Integration: parse() populates FileNode.links from file content."""
+
+ async def run():
+ content = (
+ "---\n"
+ "title: demo\n"
+ "---\n"
+ "\n"
+ "Intro paragraph with [[alpha]] and [[beta#h2]].\n"
+ "author:: [[Alice]]\n"
+ "[ref:: [[paper#chapter 1]]]\n"
+ )
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f:
+ f.write(content)
+ temp_path = f.name
+
+ try:
+ parser = DefaultFileParser()
+ file_node, _ = await parser.parse(temp_path)
+ triples = {(link.predicate, link.target_path, link.target_anchor) for link in file_node.links}
+ assert (None, "alpha", None) in triples
+ assert (None, "beta", "h2") in triples
+ assert ("author", "Alice", None) in triples
+ assert ("ref", "paper", "chapter 1") in triples
+ assert all(link.source_path == file_node.path for link in file_node.links)
+ print("✓ test_parse_links_in_file passed")
+ finally:
+ os.unlink(temp_path)
+
+ asyncio.run(run())
+
+
+def test_parse_links_empty_when_no_content():
+ """Empty file and front-matter-only file both yield no links."""
+
+ async def run():
+ # Empty file
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f:
+ empty_path = f.name
+ # Front-matter-only file
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f:
+ f.write("---\ntitle: x\n---\n")
+ fm_only_path = f.name
+
+ try:
+ parser = DefaultFileParser()
+ node1, _ = await parser.parse(empty_path)
+ node2, _ = await parser.parse(fm_only_path)
+ assert node1.links == []
+ assert node2.links == []
+ print("✓ test_parse_links_empty_when_no_content passed")
+ finally:
+ os.unlink(empty_path)
+ os.unlink(fm_only_path)
+
+ asyncio.run(run())
+
+
+def test_min_chunk_and_overlap_size():
+ """Test that minimum chunk and overlap sizes are enforced."""
+
+ async def run():
+ # These values should be clamped to minimums
+ parser = DefaultFileParser(chunk_byte_size=1, overlap_byte_size=0)
+ assert parser.chunk_byte_size == 100 # minimum
+ assert parser.overlap_byte_size == 4 # minimum
+
+ content = "test"
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f:
+ f.write(content)
+ temp_path = f.name
+
+ try:
+ _, chunks = await parser.parse(temp_path)
+ assert len(chunks) == 1
+ print("✓ test_min_chunk_and_overlap_size passed")
+ finally:
+ os.unlink(temp_path)
+
+ asyncio.run(run())
+
+
+if __name__ == "__main__":
+ test_parse_empty_file()
+ test_parse_small_file()
+ test_parse_multiline_file()
+ test_parse_chunked_file()
+ test_parse_with_custom_encoding()
+ test_file_node_properties()
+ test_file_chunk_properties()
+ test_parse_links_bare()
+ test_parse_links_with_anchor()
+ test_parse_links_alias_dropped()
+ test_parse_links_anchor_and_alias()
+ test_parse_links_predicate_simple()
+ test_parse_links_predicate_bracketed()
+ test_parse_links_predicate_bracketed_with_anchor()
+ test_parse_links_predicate_sticks_to_first()
+ test_parse_links_multiple_on_one_line()
+ test_parse_links_no_match()
+ test_parse_links_predicate_with_dash_and_digits()
+ test_parse_links_in_file()
+ test_parse_links_empty_when_no_content()
+ test_min_chunk_and_overlap_size()
+ print("\n所有测试通过!")
diff --git a/tests4/unittest/test_file_graph.py b/tests4/unittest/test_file_graph.py
new file mode 100644
index 00000000..1dc3f6f5
--- /dev/null
+++ b/tests4/unittest/test_file_graph.py
@@ -0,0 +1,333 @@
+"""Tests for FileGraph backends (LocalFileGraph + NxFileGraph)."""
+
+# pylint: disable=protected-access
+
+import asyncio
+import os
+import tempfile
+
+import pytest
+
+from reme4.components.file_graph import LocalFileGraph, NxFileGraph
+from reme4.schema import FileLink, FileNode
+
+
+class temp_chdir:
+ """Context manager to temporarily chdir into a path and restore on exit."""
+
+ def __init__(self, path):
+ self.path = path
+ self.old = None
+
+ def __enter__(self):
+ self.old = os.getcwd()
+ os.chdir(self.path)
+ return self
+
+ def __exit__(self, *exc):
+ os.chdir(self.old)
+
+
+def make_node(path: str, links: list[tuple[str, str | None]] | None = None) -> FileNode:
+ """Build a FileNode with the given outgoing (target_path, target_anchor) pairs."""
+ return FileNode(
+ path=path,
+ st_mtime=1.0,
+ links=[FileLink(source_path=path, target_path=t, target_anchor=a) for t, a in (links or [])],
+ )
+
+
+# Both backends should satisfy the same BaseFileGraph contract.
+BACKENDS = [LocalFileGraph, NxFileGraph]
+
+
+@pytest.mark.parametrize("backend_cls", BACKENDS)
+def test_upsert_and_get_nodes(backend_cls):
+ """upsert_nodes stores nodes; get_nodes returns them by path or all."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ graph = backend_cls()
+ await graph.start()
+
+ n1 = make_node("a.md", [("b.md", None)])
+ n2 = make_node("b.md")
+ await graph.upsert_nodes([n1, n2])
+
+ got_all = await graph.get_nodes()
+ assert {n.path for n in got_all} == {"a.md", "b.md"}
+
+ got_one = await graph.get_nodes(["a.md"])
+ assert len(got_one) == 1
+ assert got_one[0].path == "a.md"
+
+ got_missing = await graph.get_nodes(["nope.md"])
+ assert got_missing == []
+
+ await graph.close()
+ print(f"✓ test_upsert_and_get_nodes[{backend_cls.__name__}] passed")
+
+ asyncio.run(run())
+
+
+@pytest.mark.parametrize("backend_cls", BACKENDS)
+def test_upsert_replaces_old_links(backend_cls):
+ """Re-upserting a node with new links replaces the old outgoing edges."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ graph = backend_cls()
+ await graph.start()
+
+ await graph.upsert_nodes(
+ [
+ make_node("a.md", [("b.md", None)]),
+ make_node("b.md"),
+ make_node("c.md"),
+ ],
+ )
+ assert {lnk.target_path for lnk in await graph.get_outlinks("a.md")} == {"b.md"}
+
+ # Replace a's link target from b → c
+ await graph.upsert_nodes([make_node("a.md", [("c.md", None)])])
+ assert {lnk.target_path for lnk in await graph.get_outlinks("a.md")} == {"c.md"}
+ # b should no longer have a as an inlink
+ assert await graph.get_inlinks("b.md") == []
+ assert {lnk.source_path for lnk in await graph.get_inlinks("c.md")} == {"a.md"}
+
+ await graph.close()
+ print(f"✓ test_upsert_replaces_old_links[{backend_cls.__name__}] passed")
+
+ asyncio.run(run())
+
+
+@pytest.mark.parametrize("backend_cls", BACKENDS)
+def test_outlinks_skip_virtual_targets(backend_cls):
+ """get_outlinks only returns links pointing to real (existing) nodes."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ graph = backend_cls()
+ await graph.start()
+
+ # a links to b (real) and ghost (virtual)
+ await graph.upsert_nodes(
+ [
+ make_node("a.md", [("b.md", None), ("ghost.md", None)]),
+ make_node("b.md"),
+ ],
+ )
+
+ outs = await graph.get_outlinks("a.md")
+ targets = {lnk.target_path for lnk in outs}
+ assert targets == {"b.md"}
+
+ await graph.close()
+ print(f"✓ test_outlinks_skip_virtual_targets[{backend_cls.__name__}] passed")
+
+ asyncio.run(run())
+
+
+@pytest.mark.parametrize("backend_cls", BACKENDS)
+def test_inlinks_promotion_after_upsert(backend_cls):
+ """Edges to virtual targets become real inlinks once the target is upserted."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ graph = backend_cls()
+ await graph.start()
+
+ # b doesn't exist yet — link is pending
+ await graph.upsert_nodes([make_node("a.md", [("b.md", None)])])
+ assert await graph.get_inlinks("b.md") == [] # b not real yet
+
+ # Now create b — pending edge promotes
+ await graph.upsert_nodes([make_node("b.md")])
+ inlinks = await graph.get_inlinks("b.md")
+ assert {lnk.source_path for lnk in inlinks} == {"a.md"}
+
+ await graph.close()
+ print(f"✓ test_inlinks_promotion_after_upsert[{backend_cls.__name__}] passed")
+
+ asyncio.run(run())
+
+
+@pytest.mark.parametrize("backend_cls", BACKENDS)
+def test_delete_node_demotes_inbound(backend_cls):
+ """Deleting a node makes it virtual; sources still hold the link, but get_inlinks([deleted]) is []."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ graph = backend_cls()
+ await graph.start()
+
+ await graph.upsert_nodes(
+ [
+ make_node("a.md", [("b.md", None)]),
+ make_node("b.md"),
+ ],
+ )
+ assert {lnk.source_path for lnk in await graph.get_inlinks("b.md")} == {"a.md"}
+
+ await graph.delete_nodes(["b.md"])
+ # b is no longer a real node
+ assert await graph.get_nodes(["b.md"]) == []
+ # inlinks query for a non-real node returns []
+ assert await graph.get_inlinks("b.md") == []
+ # a's outlink to b is hidden because b is virtual
+ assert await graph.get_outlinks("a.md") == []
+
+ # Re-upsert b — pending should re-promote
+ await graph.upsert_nodes([make_node("b.md")])
+ assert {lnk.source_path for lnk in await graph.get_inlinks("b.md")} == {"a.md"}
+
+ await graph.close()
+ print(f"✓ test_delete_node_demotes_inbound[{backend_cls.__name__}] passed")
+
+ asyncio.run(run())
+
+
+@pytest.mark.parametrize("backend_cls", BACKENDS)
+def test_delete_outgoing_links_cleared(backend_cls):
+ """Deleting a source node drops its outgoing edges (no inlink left on its targets)."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ graph = backend_cls()
+ await graph.start()
+
+ await graph.upsert_nodes(
+ [
+ make_node("a.md", [("b.md", None)]),
+ make_node("b.md"),
+ ],
+ )
+ await graph.delete_nodes(["a.md"])
+
+ assert await graph.get_inlinks("b.md") == []
+
+ await graph.close()
+ print(f"✓ test_delete_outgoing_links_cleared[{backend_cls.__name__}] passed")
+
+ asyncio.run(run())
+
+
+@pytest.mark.parametrize("backend_cls", BACKENDS)
+def test_clear(backend_cls):
+ """clear() removes all nodes and edges."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ graph = backend_cls()
+ await graph.start()
+
+ await graph.upsert_nodes(
+ [
+ make_node("a.md", [("b.md", None)]),
+ make_node("b.md"),
+ ],
+ )
+ await graph.clear()
+ assert await graph.get_nodes() == []
+
+ await graph.close()
+ print(f"✓ test_clear[{backend_cls.__name__}] passed")
+
+ asyncio.run(run())
+
+
+@pytest.mark.parametrize("backend_cls", BACKENDS)
+def test_rebuild_links_idempotent(backend_cls):
+ """rebuild_links produces the same outlink/inlink view as the original upserts."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ graph = backend_cls()
+ await graph.start()
+
+ await graph.upsert_nodes(
+ [
+ make_node("a.md", [("b.md", None), ("c.md", "h")]),
+ make_node("b.md"),
+ make_node("c.md"),
+ ],
+ )
+
+ before_out = sorted((lnk.target_path, lnk.target_anchor) for lnk in await graph.get_outlinks("a.md"))
+ before_in = sorted(lnk.source_path for lnk in await graph.get_inlinks("b.md"))
+
+ await graph.rebuild_links()
+
+ after_out = sorted((lnk.target_path, lnk.target_anchor) for lnk in await graph.get_outlinks("a.md"))
+ after_in = sorted(lnk.source_path for lnk in await graph.get_inlinks("b.md"))
+
+ assert before_out == after_out
+ assert before_in == after_in
+
+ await graph.close()
+ print(f"✓ test_rebuild_links_idempotent[{backend_cls.__name__}] passed")
+
+ asyncio.run(run())
+
+
+@pytest.mark.parametrize("backend_cls", BACKENDS)
+def test_persistence_roundtrip(backend_cls):
+ """close() dumps; a fresh instance loads the same nodes from disk."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ g1 = backend_cls()
+ await g1.start()
+ await g1.upsert_nodes(
+ [
+ make_node("a.md", [("b.md", None)]),
+ make_node("b.md"),
+ ],
+ )
+ await g1.close() # triggers dump
+
+ g2 = backend_cls()
+ await g2.start() # triggers load
+ paths = sorted(n.path for n in await g2.get_nodes())
+ assert paths == ["a.md", "b.md"]
+ # Inlink relationship should also be reconstructable.
+ assert {lnk.source_path for lnk in await g2.get_inlinks("b.md")} == {"a.md"}
+ await g2.close()
+ print(f"✓ test_persistence_roundtrip[{backend_cls.__name__}] passed")
+
+ asyncio.run(run())
+
+
+@pytest.mark.parametrize("backend_cls", BACKENDS)
+def test_get_nodes_empty_inputs(backend_cls):
+ """get_nodes([]) returns []; get_nodes(None) returns all."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ graph = backend_cls()
+ await graph.start()
+
+ await graph.upsert_nodes([make_node("a.md")])
+ assert await graph.get_nodes([]) == []
+ assert len(await graph.get_nodes(None)) == 1
+
+ await graph.close()
+ print(f"✓ test_get_nodes_empty_inputs[{backend_cls.__name__}] passed")
+
+ asyncio.run(run())
+
+
+if __name__ == "__main__":
+ print("\n=== FileGraph Tests ===")
+ for backend in BACKENDS:
+ test_upsert_and_get_nodes(backend)
+ test_upsert_replaces_old_links(backend)
+ test_outlinks_skip_virtual_targets(backend)
+ test_inlinks_promotion_after_upsert(backend)
+ test_delete_node_demotes_inbound(backend)
+ test_delete_outgoing_links_cleared(backend)
+ test_clear(backend)
+ test_rebuild_links_idempotent(backend)
+ test_persistence_roundtrip(backend)
+ test_get_nodes_empty_inputs(backend)
+ print("\n所有测试通过!")
diff --git a/tests4/unittest/test_file_store.py b/tests4/unittest/test_file_store.py
new file mode 100644
index 00000000..e59e305b
--- /dev/null
+++ b/tests4/unittest/test_file_store.py
@@ -0,0 +1,330 @@
+"""Tests for LocalFileStore."""
+
+# pylint: disable=protected-access
+
+import asyncio
+import os
+import tempfile
+import warnings
+
+from reme4.components.file_store import LocalFileStore
+from reme4.schema import FileChunk, FileNode
+
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba")
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources")
+
+
+class temp_chdir:
+ """Context manager to temporarily chdir into a path and restore on exit."""
+
+ def __init__(self, path):
+ self.path = path
+ self.old = None
+
+ def __enter__(self):
+ self.old = os.getcwd()
+ os.chdir(self.path)
+ return self
+
+ def __exit__(self, *exc):
+ os.chdir(self.old)
+
+
+async def make_store(store_name: str = "test_store", **kwargs) -> LocalFileStore:
+ """Build a started LocalFileStore with embedding disabled (no OpenAI dep)."""
+ store = LocalFileStore(store_name=store_name, embedding_model="", **kwargs)
+ await store.start()
+ return store
+
+
+def make_file(
+ path: str,
+ text: str,
+ chunk_count: int = 1,
+) -> tuple[FileNode, list[FileChunk]]:
+ """Build a (FileNode, [FileChunk]) tuple ready for upsert_file."""
+ chunks = [
+ FileChunk(id=f"{path}::chunk{i}", path=path, text=f"{text} part{i}", start_line=i, end_line=i + 1)
+ for i in range(chunk_count)
+ ]
+ node = FileNode(path=path, st_mtime=1.0, chunk_ids=[c.id for c in chunks])
+ return node, chunks
+
+
+def test_upsert_single_file():
+ """upsert_file with a single (node, chunks) tuple stores chunks and node."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ store = await make_store()
+
+ node, chunks = make_file("a.md", "hello world", chunk_count=2)
+ await store.upsert_file((node, chunks))
+
+ # Chunks landed in memory
+ assert len(store.file_chunks) == 2
+ assert {c.path for c in store.file_chunks.values()} == {"a.md"}
+ # Node landed in graph
+ nodes = await store.file_graph.get_nodes(["a.md"])
+ assert len(nodes) == 1
+ assert sorted(nodes[0].chunk_ids) == sorted([c.id for c in chunks])
+
+ await store.close()
+ print("✓ test_upsert_single_file passed")
+
+ asyncio.run(run())
+
+
+def test_upsert_multiple_files():
+ """upsert_file accepts a list of tuples and indexes them all."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ store = await make_store()
+
+ files = [make_file("a.md", "alpha"), make_file("b.md", "beta")]
+ await store.upsert_file(files)
+
+ assert len(store.file_chunks) == 2
+ paths = {n.path for n in await store.file_graph.get_nodes()}
+ assert paths == {"a.md", "b.md"}
+
+ await store.close()
+ print("✓ test_upsert_multiple_files passed")
+
+ asyncio.run(run())
+
+
+def test_upsert_replaces_old_chunks():
+ """Re-upserting the same path points the node at the new chunk set."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ store = await make_store()
+
+ n1, c1 = make_file("a.md", "v1", chunk_count=2)
+ await store.upsert_file((n1, c1))
+
+ # Different chunks for the same path
+ n2 = FileNode(path="a.md", st_mtime=2.0)
+ c2 = [FileChunk(id="a.md::new", path="a.md", text="v2 only", start_line=0, end_line=1)]
+ n2.chunk_ids = [c.id for c in c2]
+ await store.upsert_file((n2, c2))
+
+ # The node now references the new chunk set, not the old one.
+ nodes = await store.file_graph.get_nodes(["a.md"])
+ assert nodes[0].chunk_ids == ["a.md::new"]
+ assert "a.md::new" in store.file_chunks
+
+ await store.close()
+ print("✓ test_upsert_replaces_old_chunks passed")
+
+ asyncio.run(run())
+
+
+def test_delete_by_path_single():
+ """delete_by_path drops chunks and the node entry."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ store = await make_store()
+
+ await store.upsert_file([make_file("a.md", "alpha"), make_file("b.md", "beta")])
+ await store.delete_by_path("a.md")
+
+ assert all(c.path != "a.md" for c in store.file_chunks.values())
+ assert {n.path for n in await store.file_graph.get_nodes()} == {"b.md"}
+
+ await store.close()
+ print("✓ test_delete_by_path_single passed")
+
+ asyncio.run(run())
+
+
+def test_delete_by_path_list():
+ """delete_by_path accepts a list of paths."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ store = await make_store()
+
+ await store.upsert_file(
+ [
+ make_file("a.md", "alpha"),
+ make_file("b.md", "beta"),
+ make_file("c.md", "gamma"),
+ ],
+ )
+ await store.delete_by_path(["a.md", "b.md"])
+
+ assert {n.path for n in await store.file_graph.get_nodes()} == {"c.md"}
+ assert all(c.path == "c.md" for c in store.file_chunks.values())
+
+ await store.close()
+ print("✓ test_delete_by_path_list passed")
+
+ asyncio.run(run())
+
+
+def test_delete_by_path_missing_is_noop():
+ """Deleting a nonexistent path is a no-op."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ store = await make_store()
+
+ await store.upsert_file(make_file("a.md", "alpha"))
+ before = len(store.file_chunks)
+ await store.delete_by_path("ghost.md")
+ assert len(store.file_chunks) == before
+
+ await store.close()
+ print("✓ test_delete_by_path_missing_is_noop passed")
+
+ asyncio.run(run())
+
+
+def test_clear():
+ """clear() empties chunks and the file graph."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ store = await make_store()
+
+ await store.upsert_file([make_file("a.md", "alpha"), make_file("b.md", "beta")])
+ await store.clear()
+
+ assert store.file_chunks == {}
+ assert await store.file_graph.get_nodes() == []
+
+ await store.close()
+ print("✓ test_clear passed")
+
+ asyncio.run(run())
+
+
+def test_keyword_search():
+ """keyword_search returns matching chunks ranked by BM25 score."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ store = await make_store()
+
+ await store.upsert_file(
+ [
+ make_file("a.md", "python programming language"),
+ make_file("b.md", "java programming language"),
+ make_file("c.md", "python data analysis"),
+ ],
+ )
+
+ results = await store.keyword_search("python", limit=5, search_filter={})
+ paths = {r.path for r in results}
+ assert "a.md" in paths or "c.md" in paths
+ # Each result should carry a keyword score.
+ for r in results:
+ assert r.scores.get("keyword", 0) > 0
+
+ await store.close()
+ print("✓ test_keyword_search passed")
+
+ asyncio.run(run())
+
+
+def test_keyword_search_empty_query():
+ """Empty/whitespace queries return no results."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ store = await make_store()
+ await store.upsert_file(make_file("a.md", "hello"))
+
+ assert await store.keyword_search("", limit=5, search_filter={}) == []
+ assert await store.keyword_search(" ", limit=5, search_filter={}) == []
+
+ await store.close()
+ print("✓ test_keyword_search_empty_query passed")
+
+ asyncio.run(run())
+
+
+def test_vector_search_disabled_returns_empty():
+ """Without an embedding model, vector_search returns []."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ store = await make_store()
+ await store.upsert_file(make_file("a.md", "hello"))
+
+ assert store.embedding_model is None
+ assert await store.vector_search("hello", limit=5, search_filter={}) == []
+
+ await store.close()
+ print("✓ test_vector_search_disabled_returns_empty passed")
+
+ asyncio.run(run())
+
+
+def test_persistence_roundtrip():
+ """close() dumps chunks; a fresh store loads them from disk."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ s1 = await make_store()
+ await s1.upsert_file([make_file("a.md", "alpha"), make_file("b.md", "beta")])
+ await s1.close()
+
+ s2 = await make_store()
+ assert {c.path for c in s2.file_chunks.values()} == {"a.md", "b.md"}
+ # Graph should also be persisted independently via its own dump.
+ assert {n.path for n in await s2.file_graph.get_nodes()} == {"a.md", "b.md"}
+ await s2.close()
+ print("✓ test_persistence_roundtrip passed")
+
+ asyncio.run(run())
+
+
+def test_rebuild_links_delegates_to_graph():
+ """rebuild_links() on the store delegates to the underlying file_graph."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ store = await make_store()
+
+ from reme4.schema import FileLink
+
+ node = FileNode(
+ path="a.md",
+ st_mtime=1.0,
+ links=[FileLink(source_path="a.md", target_path="b.md")],
+ )
+ chunks = [FileChunk(id="a::1", path="a.md", text="x", start_line=0, end_line=1)]
+ node.chunk_ids = [c.id for c in chunks]
+ await store.upsert_file((node, chunks))
+ await store.upsert_file(make_file("b.md", "beta"))
+
+ await store.rebuild_links()
+ inlinks = await store.get_inlinks("b.md")
+ assert {lnk.source_path for lnk in inlinks} == {"a.md"}
+
+ await store.close()
+ print("✓ test_rebuild_links_delegates_to_graph passed")
+
+ asyncio.run(run())
+
+
+if __name__ == "__main__":
+ print("\n=== LocalFileStore Tests ===")
+ test_upsert_single_file()
+ test_upsert_multiple_files()
+ test_upsert_replaces_old_chunks()
+ test_delete_by_path_single()
+ test_delete_by_path_list()
+ test_delete_by_path_missing_is_noop()
+ test_clear()
+ test_keyword_search()
+ test_keyword_search_empty_query()
+ test_vector_search_disabled_returns_empty()
+ test_persistence_roundtrip()
+ test_rebuild_links_delegates_to_graph()
+ print("\n所有测试通过!")
diff --git a/tests4/unittest/test_file_watcher.py b/tests4/unittest/test_file_watcher.py
new file mode 100644
index 00000000..f7008567
--- /dev/null
+++ b/tests4/unittest/test_file_watcher.py
@@ -0,0 +1,349 @@
+"""Tests for LiteFileWatcher (excluding the awatch main loop)."""
+
+# pylint: disable=protected-access
+
+import asyncio
+import os
+import tempfile
+import warnings
+from pathlib import Path
+
+from watchfiles import Change
+
+from reme4.components.file_parser import DefaultFileParser
+from reme4.components.file_store import LocalFileStore
+from reme4.components.file_watcher import LiteFileWatcher
+
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba")
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources")
+
+
+class temp_chdir:
+ """Context manager to temporarily chdir into a path and restore on exit."""
+
+ def __init__(self, path):
+ self.path = path
+ self.old = None
+
+ def __enter__(self):
+ self.old = os.getcwd()
+ os.chdir(self.path)
+ return self
+
+ def __exit__(self, *exc):
+ os.chdir(self.old)
+
+
+async def make_watcher(watch_paths: list[str] | str = "vault", **kwargs) -> LiteFileWatcher:
+ """Build a LiteFileWatcher with real (started) file_store/file_parser, but no background loop.
+
+ We replace the bind() Dependency placeholders with concrete instances and start them
+ manually, so tests can call update_store / on_* directly without the background task.
+ """
+ watcher = LiteFileWatcher(watch_paths=watch_paths, **kwargs)
+ fs = LocalFileStore(store_name="test_store", embedding_model="")
+ parser = DefaultFileParser()
+ await fs.start()
+ await parser.start()
+ watcher.file_store = fs
+ watcher.file_parser = parser
+ return watcher
+
+
+async def teardown_watcher(watcher: LiteFileWatcher) -> None:
+ """Close the manually-started subcomponents."""
+ await watcher.file_parser.close()
+ await watcher.file_store.close()
+
+
+def write_file(path: Path, content: str = "x") -> Path:
+ """Create or overwrite a file and return the path."""
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(content, encoding="utf-8")
+ return path
+
+
+def test_watch_filter_default_md():
+ """Default suffix_filters=['md'] passes .md files and rejects others."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ (Path(tmpdir) / "vault").mkdir()
+ watcher = await make_watcher()
+
+ assert watcher.watch_filter(Change.added, "/x/foo.md")
+ assert not watcher.watch_filter(Change.added, "/x/foo.txt")
+ assert not watcher.watch_filter(Change.added, "/x/foo")
+
+ print("✓ test_watch_filter_default_md passed")
+
+ asyncio.run(run())
+
+
+def test_watch_filter_custom_suffix():
+ """Custom suffix_filters override the default."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ (Path(tmpdir) / "vault").mkdir()
+ watcher = await make_watcher(suffix_filters=["txt", ".rst"])
+
+ assert watcher.watch_filter(Change.added, "/x/foo.txt")
+ assert watcher.watch_filter(Change.added, "/x/foo.rst")
+ assert not watcher.watch_filter(Change.added, "/x/foo.md")
+
+ print("✓ test_watch_filter_custom_suffix passed")
+
+ asyncio.run(run())
+
+
+def test_watch_filter_no_filter_passes_all():
+ """When suffix_filters is empty (set after init), watch_filter passes everything."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ (Path(tmpdir) / "vault").mkdir()
+ watcher = await make_watcher()
+ # Constructor coerces [] / None → ["md"]; clear it directly to exercise the no-filter branch.
+ watcher.suffix_filters = []
+
+ assert watcher.watch_filter(Change.added, "/x/foo")
+ assert watcher.watch_filter(Change.added, "/x/foo.md")
+
+ print("✓ test_watch_filter_no_filter_passes_all passed")
+
+ asyncio.run(run())
+
+
+def test_relative_and_absolute_path_helpers():
+ """_get_relative_path strips working_path; _get_absolute_path resolves against it."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ (Path(tmpdir) / "vault").mkdir()
+ watcher = await make_watcher()
+
+ # Use watcher.working_path to match the same realpath form (macOS /var ↔ /private/var).
+ abs_in = (watcher.working_path / "vault" / "a.md").absolute()
+ assert watcher._get_relative_path(abs_in) == "vault/a.md"
+
+ # Path outside working_path → returns absolute
+ outside = Path("/opt/elsewhere/x.md").absolute()
+ assert watcher._get_relative_path(outside) == str(outside)
+
+ # _get_absolute_path: relative resolves under working_path
+ assert watcher._get_absolute_path("vault/a.md") == watcher.working_path / "vault/a.md"
+ # absolute stays absolute
+ assert watcher._get_absolute_path(str(abs_in)) == abs_in
+
+ print("✓ test_relative_and_absolute_path_helpers passed")
+
+ asyncio.run(run())
+
+
+def test_scan_existing_files_finds_md_recursive():
+ """scan_existing_files returns md files under watch_paths, recursively."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ vault = Path(tmpdir) / "vault"
+ write_file(vault / "a.md", "alpha")
+ write_file(vault / "sub" / "b.md", "beta")
+ write_file(vault / "ignore.txt", "skip") # filtered by suffix
+ watcher = await make_watcher()
+
+ files = await watcher.scan_existing_files()
+ rels = set(files.keys())
+ assert "vault/a.md" in rels
+ assert "vault/sub/b.md" in rels
+ assert "vault/ignore.txt" not in rels
+
+ print("✓ test_scan_existing_files_finds_md_recursive passed")
+
+ asyncio.run(run())
+
+
+def test_scan_existing_files_non_recursive():
+ """recursive=False only scans direct children."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ vault = Path(tmpdir) / "vault"
+ write_file(vault / "a.md", "alpha")
+ write_file(vault / "sub" / "b.md", "beta")
+ watcher = await make_watcher(recursive=False)
+
+ files = await watcher.scan_existing_files()
+ rels = set(files.keys())
+ assert "vault/a.md" in rels
+ assert "vault/sub/b.md" not in rels
+
+ print("✓ test_scan_existing_files_non_recursive passed")
+
+ asyncio.run(run())
+
+
+def test_on_added_indexes_files():
+ """on_added parses files and writes them into the file_store."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ vault = Path(tmpdir) / "vault"
+ write_file(vault / "a.md", "hello")
+ watcher = await make_watcher()
+ try:
+ await watcher.on_added(["vault/a.md"])
+ nodes = await watcher.file_store.file_graph.get_nodes()
+ assert {n.path for n in nodes} == {"vault/a.md"}
+ finally:
+ await teardown_watcher(watcher)
+ print("✓ test_on_added_indexes_files passed")
+
+ asyncio.run(run())
+
+
+def test_on_modified_replaces_node():
+ """on_modified re-parses and updates the node entry for an existing path."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ vault = Path(tmpdir) / "vault"
+ file_path = write_file(vault / "a.md", "v1")
+ watcher = await make_watcher()
+ try:
+ await watcher.on_added(["vault/a.md"])
+ node_before = (await watcher.file_store.file_graph.get_nodes(["vault/a.md"]))[0]
+ # Bump mtime + content
+ file_path.write_text("v2 different content", encoding="utf-8")
+ os.utime(file_path, (node_before.st_mtime + 10, node_before.st_mtime + 10))
+
+ await watcher.on_modified(["vault/a.md"])
+ node_after = (await watcher.file_store.file_graph.get_nodes(["vault/a.md"]))[0]
+ assert node_after.st_mtime > node_before.st_mtime
+ finally:
+ await teardown_watcher(watcher)
+ print("✓ test_on_modified_replaces_node passed")
+
+ asyncio.run(run())
+
+
+def test_on_deleted_removes_node():
+ """on_deleted removes the node from the store regardless of file presence."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ vault = Path(tmpdir) / "vault"
+ write_file(vault / "a.md", "hello")
+ watcher = await make_watcher()
+ try:
+ await watcher.on_added(["vault/a.md"])
+ assert {n.path for n in await watcher.file_store.file_graph.get_nodes()} == {"vault/a.md"}
+
+ await watcher.on_deleted(["vault/a.md"])
+ assert await watcher.file_store.file_graph.get_nodes() == []
+ finally:
+ await teardown_watcher(watcher)
+ print("✓ test_on_deleted_removes_node passed")
+
+ asyncio.run(run())
+
+
+def test_update_store_initial_add():
+ """First update_store run on a fresh store reports all files as added."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ vault = Path(tmpdir) / "vault"
+ write_file(vault / "a.md", "alpha")
+ write_file(vault / "b.md", "beta")
+ watcher = await make_watcher()
+ try:
+ counts = await watcher.update_store(dump=False)
+ assert counts == {"added": 2, "modified": 0, "deleted": 0}
+ paths = {n.path for n in await watcher.file_store.file_graph.get_nodes()}
+ assert paths == {"vault/a.md", "vault/b.md"}
+ finally:
+ await teardown_watcher(watcher)
+ print("✓ test_update_store_initial_add passed")
+
+ asyncio.run(run())
+
+
+def test_update_store_detects_modify_and_delete():
+ """update_store distinguishes modified vs deleted vs added on a second pass."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ vault = Path(tmpdir) / "vault"
+ a = write_file(vault / "a.md", "alpha")
+ b = write_file(vault / "b.md", "beta")
+ watcher = await make_watcher()
+ try:
+ # Initial sync to seed the store.
+ await watcher.update_store(dump=False)
+
+ # Modify a (bump mtime), delete b, add c.
+ a.write_text("alpha-v2", encoding="utf-8")
+ os.utime(a, (9_999_999_999, 9_999_999_999))
+ b.unlink()
+ write_file(vault / "c.md", "gamma")
+
+ counts = await watcher.update_store(dump=False)
+ assert counts == {"added": 1, "modified": 1, "deleted": 1}
+ paths = {n.path for n in await watcher.file_store.file_graph.get_nodes()}
+ assert paths == {"vault/a.md", "vault/c.md"}
+ finally:
+ await teardown_watcher(watcher)
+ print("✓ test_update_store_detects_modify_and_delete passed")
+
+ asyncio.run(run())
+
+
+def test_update_store_no_changes():
+ """A second sync over an unchanged tree reports zero counts."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ vault = Path(tmpdir) / "vault"
+ write_file(vault / "a.md", "alpha")
+ watcher = await make_watcher()
+ try:
+ await watcher.update_store(dump=False)
+ counts = await watcher.update_store(dump=False)
+ assert counts == {"added": 0, "modified": 0, "deleted": 0}
+ finally:
+ await teardown_watcher(watcher)
+ print("✓ test_update_store_no_changes passed")
+
+ asyncio.run(run())
+
+
+def test_missing_watch_path_filtered():
+ """watch_paths entries that don't exist are dropped from self.watch_paths."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
+ (Path(tmpdir) / "vault").mkdir()
+ watcher = await make_watcher(watch_paths=["vault", "ghost"])
+ assert [p.name for p in watcher.watch_paths] == ["vault"]
+ print("✓ test_missing_watch_path_filtered passed")
+
+ asyncio.run(run())
+
+
+if __name__ == "__main__":
+ print("\n=== LiteFileWatcher Tests ===")
+ test_watch_filter_default_md()
+ test_watch_filter_custom_suffix()
+ test_watch_filter_no_filter_passes_all()
+ test_relative_and_absolute_path_helpers()
+ test_scan_existing_files_finds_md_recursive()
+ test_scan_existing_files_non_recursive()
+ test_on_added_indexes_files()
+ test_on_modified_replaces_node()
+ test_on_deleted_removes_node()
+ test_update_store_initial_add()
+ test_update_store_detects_modify_and_delete()
+ test_update_store_no_changes()
+ test_missing_watch_path_filtered()
+ print("\n所有测试通过!")
diff --git a/tests4/unittest/test_tokenizer.py b/tests4/unittest/test_tokenizer.py
new file mode 100644
index 00000000..5dc99308
--- /dev/null
+++ b/tests4/unittest/test_tokenizer.py
@@ -0,0 +1,169 @@
+"""Tests for Tokenizers."""
+
+import asyncio
+
+from reme4.components.tokenizer import JiebaTokenizer, RegexTokenizer
+
+
+async def compare_tokenizers(texts: list[str], filter_stopwords: bool = False, name: str = ""):
+ """Compare both tokenizers on same input."""
+ jieba = JiebaTokenizer(filter_stopwords=filter_stopwords)
+ regex = RegexTokenizer(filter_stopwords=filter_stopwords)
+
+ await jieba.start()
+ await regex.start()
+
+ jieba_result = jieba.tokenize(texts)
+ regex_result = regex.tokenize(texts)
+
+ print(f"\n--- {name} ---")
+ print(f"输入: {texts}")
+ print(f"Jieba: {jieba_result}")
+ print(f"Regex: {regex_result}")
+
+ await jieba.close()
+ await regex.close()
+
+ return jieba_result, regex_result
+
+
+def test_basic_chinese():
+ """Test basic Chinese text."""
+
+ async def run():
+ jieba_result, regex_result = await compare_tokenizers(
+ ["我爱北京天安门", "今天天气很好"],
+ name="纯中文",
+ )
+
+ assert "北京" in jieba_result[0] or "天安门" in jieba_result[0]
+ assert "我" in regex_result[0]
+ print("✓ test_basic_chinese passed")
+
+ asyncio.run(run())
+
+
+def test_basic_english():
+ """Test basic English text."""
+
+ async def run():
+ _, regex_result = await compare_tokenizers(
+ ["I love Beijing very much"],
+ name="英文",
+ )
+
+ assert "love" in regex_result[0]
+ assert "beijing" in regex_result[0]
+ print("✓ test_basic_english passed")
+
+ asyncio.run(run())
+
+
+def test_mixed_chinese_english():
+ """Test mixed Chinese-English text."""
+
+ async def run():
+ jieba_result, regex_result = await compare_tokenizers(
+ ["我用 Python 学习 machine learning 和 iPhone15 Pro。"],
+ name="中英混合",
+ )
+
+ assert "python" in jieba_result[0]
+ assert "python" in regex_result[0]
+ print("✓ test_mixed_chinese_english passed")
+
+ asyncio.run(run())
+
+
+def test_open_example():
+ """Test the 'open' example."""
+
+ async def run():
+ jieba_result, regex_result = await compare_tokenizers(
+ ["我觉得open很好呀，能分好次吗？"],
+ name="'open' 案例",
+ )
+
+ # open 保持完整
+ assert "open" in jieba_result[0]
+ assert "open" in regex_result[0]
+
+ # Regex 中文按字拆分
+ assert "我" in regex_result[0]
+ assert "很" in regex_result[0]
+
+ print("✓ test_open_example passed")
+
+ asyncio.run(run())
+
+
+def test_with_stopwords():
+ """Test with stopwords filtering."""
+
+ async def run():
+ jieba_result, regex_result = await compare_tokenizers(
+ ["我觉得open很好呀，能分好次吗？"],
+ filter_stopwords=True,
+ name="停用词过滤",
+ )
+
+ # 停用词被过滤
+ assert "吗" not in jieba_result[0]
+ assert "吗" not in regex_result[0]
+ assert "的" not in jieba_result[0]
+
+ print("✓ test_with_stopwords passed")
+
+ asyncio.run(run())
+
+
+def test_multiple_texts():
+ """Test multiple texts at once."""
+
+ async def run():
+ texts = [
+ "我爱北京天安门",
+ "I love Python programming",
+ "今天学习 machine learning",
+ ]
+ jieba_result, regex_result = await compare_tokenizers(texts, name="多个文本")
+
+ assert len(jieba_result) == 3
+ assert len(regex_result) == 3
+ print("✓ test_multiple_texts passed")
+
+ asyncio.run(run())
+
+
+def test_tokenizer_lifecycle():
+ """Test tokenizer start/close lifecycle."""
+
+ async def run():
+ tokenizer = JiebaTokenizer(filter_stopwords=True)
+
+ assert not tokenizer.is_started
+ assert len(tokenizer.stopwords) == 0
+
+ await tokenizer.start()
+ assert tokenizer.is_started
+ assert len(tokenizer.stopwords) > 0
+
+ await tokenizer.close()
+ assert not tokenizer.is_started
+ assert len(tokenizer.stopwords) == 0
+
+ print("✓ test_tokenizer_lifecycle passed")
+
+ asyncio.run(run())
+
+
+if __name__ == "__main__":
+ print("\n=== Tokenizer Tests ===")
+ test_basic_chinese()
+ test_basic_english()
+ test_mixed_chinese_english()
+ test_open_example()
+ test_with_stopwords()
+ test_multiple_texts()
+ test_tokenizer_lifecycle()
+ print("\n所有测试通过!")
From fdc36a22bc7a886669345cd658c6d07a46c2a4ab Mon Sep 17 00:00:00 2001
From: jinliyl <6469360+jinliyl@users.noreply.github.com>
Date: Sun, 17 May 2026 18:37:12 +0800
Subject: [PATCH 10/16] docs(cli): add comprehensive CLI commands documentation
(#237)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* docs(cli): add comprehensive CLI commands documentation
- Document CLI entry point and argument parsing mechanism
- Add detailed command reference with parameters and behaviors
- Include usage examples for common operations like start, search, and reindex
- Describe backend options and service configuration overrides
- Explain local vs server-side command execution patterns
- Provide table format documentation for all available actions
* docs(reme_design): update CLI command documentation with detailed action descriptions
- Rename section from "CLI 指令" to "基础Job" and add author attribution
- Add comprehensive table documenting all available actions with parameters and behaviors
- Include detailed explanations for input/output parameters, defaults, and internal workflows
- Update example usage commands with proper parameter passing syntax
- Add metadata information for each action including health checks and component details
- Clarify the difference between local actions and server-forwarded actions
- Document the new list action that intercepts at client side without forwarding to server
---
docs4/reme_design.md | 66 +++++++++++++++++++++++++++++++++++++-------
1 file changed, 56 insertions(+), 10 deletions(-)
diff --git a/docs4/reme_design.md b/docs4/reme_design.md
index 66acb982..15e7ee28 100644
--- a/docs4/reme_design.md
+++ b/docs4/reme_design.md
@@ -12,17 +12,63 @@ reme4 version
# 基础Job
@jinli
-说明：📥 输入参数 ｜ 📤 输出 ｜ ⭐ 必填 ｜ 🎚️ 默认值 ｜ 🛠️ 内部行为
-| 分类 | 能力 (register name) | 参数 & 行为 |
-|-----------|--------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| 🌐 通用 | 🆘 `help` (`help_step`) | 📥 无 ｜ 📤 `answer` 一行一个 job：`🛠️ \`{name}\` — {description} 📥 {params}`，参数渲染为 `name:type*`(必填) / `name:type={default}` / `name:type` ｜ 📊 `metadata.job_count` ｜ 🛠️ 自动跳过名为 `help` 的 job |
-| 🌐 通用 | 🩺 `health_check` (`health_check_step`) | 📥 无 ｜ 📤 `answer = "✅/❌ ReMe v{version} - healthy/unhealthy"` ｜ 📊 `metadata.health = {version, healthy, components}` ｜ 🧩 覆盖组件：`embedding_model`(🟢 is_started/is_healthy/model_name/dimensions/cache_size/memory) · `file_graph`(🕸️ n_nodes/n_edges/n_virtual\|n_pending/memory) · `file_store`(📦 n_chunks/n_chunks_with_embedding/memory) · `file_watcher`(👀 background_running/watch_paths) · `keyword_index`(🔤 n_docs/vocab_size/memory) ｜ 🛠️ deep sizeof（含 numpy.nbytes），未启动 / 后台未跑 / embedding 不健康 → ❌ |
-| 🌐 通用 | 🏷️ `version` (`version_step`) | 📥 无 ｜ 📤 `answer = reme4.__version__` ｜ 📊 `metadata.version` |
-| 🌐 通用 | 🔄 `reindex` (`reindex_step`) | 📥 无 ｜ 📤 `answer = "🔄 Reindexed {added} file(s)"` ｜ 📊 `metadata.counts = {added, ...}` ｜ 🛠️ 流程：`file_watcher.close()` → `file_store.clear()` → `file_watcher.update_store()` → `file_watcher.start()`（finally 保证重启） |
-| 🔎 search | 🔍 `search` (`search_step`) | 📥 `query:str` ⭐ ｜ 🎚️ `limit:int=5`(>0) ｜ 🎚️ `min_score:float=0.0` ｜ ⚖️ `vector_weight:float=0.7` ∈[0,1]（keyword 权 = 1-vw）｜ 🔀 `candidate_multiplier:float=3.0`（candidates = min(200, limit×mult)）｜ 🔗 `expand_links:bool=True` ｜ 🔢 `max_links_per_direction:int=10` ｜ 🎚️ `search_filter:dict={}` ｜ 📤 `answer` 每命中一行 `path:start-end [score=… vector=… keyword=…] text` + 缩进的 `→ outlinks (n)` / `← inlinks (n)` + `via predicate=… anchor=#…` ｜ 📊 `metadata.results` / `metadata.link_expansion` / `metadata.counts={vector,keyword,returned,hybrid}` ｜ 🛠️ 并行 `vector_search` + `keyword_search` → RRF 融合（K=60，按 chunk.id 合并）→ `min_score` 过滤 → `limit` 截断 → 邻居 meta 注入 |
-| 🧪 demo | 🪄 `demo_echo` (`demo_echo_step1` + `step2`) | 📥 `query:str=""` ｜ 🎚️ `min_score:float=0.5` ｜ 🛠️ step1：`processed_query = query.strip().lower()`，`adjusted_min_score = min_score * 0.9`，写回 context ｜ 📤 step2：`answer = "echo: {processed_query} (min_score={adjusted_min_score})"` ｜ 📊 `metadata = {step, query, min_score, processed_query, adjusted_min_score}` |
-| 🌊 demo | 🌊 `stream_demo` (`stream_demo_step1` + `step2`) | 📥 `query:str=""` ｜ 🎚️ `repeat:int=10` ｜ 🎚️ `interval:float=0.1`（秒/字符）｜ 🛠️ step1：`stream_text = query * repeat` 写回 context ｜ 📤 step2：按字符 `add_stream_string(ch, ChunkEnum.CONTENT)` 流式输出，`asyncio.sleep(interval)` 节流 |
+入口：`reme4/reme.py::main()` → `parse_args(*sys.argv[1:])` 解析首个位置参数为 `action`，后续 `key=value` 解析为 kwargs（支持
+`service.port=8080` 的 dot notation；自动剥离 `--` / `-` 前缀；值会做 bool / int / float / JSON 转换）。
+
+调用模式：
+
+- `start`：本地启动 `ReMe(Application)` 服务（不经过 client）
+- `find_reme`：本地探测正在运行的 reme，不调用服务
+- `list`：在 client 端拦截，不转发到服务端，直接返回 action 目录
+- 其他 action：通过 `call_server(action, **kwargs)` → `R.get(ComponentEnum.CLIENT, backend)` 实例化客户端并流式打印（任意未列出的
+ step register name 都按本规则透传）
+
+通用可选参数 `backend:str=http`（取值 `http` / `mcp`，对应 `reme4/components/client/{http_client,mcp_client}.py` 中
+`@R.register` 注册名）；服务端默认 host/port 见 `reme4/constants.py`，可由 `start` 端通过 `service.host=` / `service.port=`
+覆盖。
+
+说明：📥 输入参数 ｜ 📤 输出 ｜ ⭐ 必填 ｜ 🎚️ 默认值 ｜ 🛠️ 内部行为 ｜ 📊 metadata
+
+| 分类 | 指令 (register name) | 入口 | 参数 & 行为 |
+|------------|--------------------------------------------------|-------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 🚀 本地 | 🟢 `start` | `reme.py:30` → `ReMe(**kwargs).run_app()` | 📥 可选 `config=`（默认加载 `reme4/config/default.yaml`，`.yaml/.yml/.json` 都支持，含 `${ENV:-default}` 占位符）｜ 可选 `service.host=` / `service.port=` 等任意 dot-notation 覆盖 ｜ 🛠️ 流程：`load_env()` → `resolve_app_config(**kwargs)` deep merge → `precheck_start(svc)`（`utils/service_utils.py:72`：目标 host:port 已有 reme → 打印 `reme already running ...` 直接返回；端口被其他进程占用 → stderr 提示 `port {port} occupied. Start on another port: reme4 start service.port=` 并 `sys.exit(1)`）→ 启动服务 |
+| 🚀 本地 | 🧭 `find_reme` | `reme.py:36` → `utils/service_utils.py:89` | 📥 无 ｜ 📤 发现服务则 stdout 打印 `HOST={host} PORT={port} PID={pid or 'unknown'}`；未发现则 stderr 提示 `reme not started. Try: reme start` 并 `sys.exit(1)` ｜ 🛠️ 流程：先探 `REME_DEFAULT_HOST:REME_DEFAULT_PORT`（`health_check` 命中算 `reme`），再 `pgrep -af "reme.* start"` 扫描其他端口 |
+| 🛰️ 客户端 | 📜 `list` | `components/client/base_client.py:36` | 📥 无 ｜ 📤 服务端可用 action 目录（JSON，`indent=2 ensure_ascii=False`）｜ 🛠️ 在 `BaseClient.__call__` 中拦截，不进入 `_execute`，直接调用 `list_actions()`（HTTP/MCP backend 各自实现） |
+| 🌐 通用 step | 🆘 `help` (`help_step`) | `call_server("help")` | 📥 无 ｜ 📤 `answer` 一行一个 job：`🛠️ \`{name}\` — {description} 📥 {params}`，参数渲染为 `name:type*`(必填) / `name:type={default}` / `name:type` ｜ 📊 `metadata.job_count` ｜ 🛠️ 自动跳过名为 `help` 的 job |
+| 🌐 通用 step | 🩺 `health_check` (`health_check_step`) | `call_server("health_check")` | 📥 无 ｜ 📤 `answer = "✅/❌ ReMe v{version} - healthy/unhealthy"` ｜ 📊 `metadata.health = {version, healthy, components}` ｜ 🧩 覆盖组件：`embedding_model`(🟢 is_started/is_healthy/model_name/dimensions/cache_size/memory) · `file_graph`(🕸️ n_nodes/n_edges/n_virtual\|n_pending/memory) · `file_store`(📦 n_chunks/n_chunks_with_embedding/memory) · `file_watcher`(👀 background_running/watch_paths) · `keyword_index`(🔤 n_docs/vocab_size/memory) ｜ 🛠️ deep sizeof（含 numpy.nbytes），未启动 / 后台未跑 / embedding 不健康 → ❌ |
+| 🌐 通用 step | 🏷️ `version` (`version_step`) | `call_server("version")` | 📥 无 ｜ 📤 `answer = reme4.__version__` ｜ 📊 `metadata.version` |
+| 🌐 通用 step | 🔄 `reindex` (`reindex_step`) | `call_server("reindex")` | 📥 无 ｜ 📤 `answer = "🔄 Reindexed {added} file(s)"` ｜ 📊 `metadata.counts = {added, ...}` ｜ 🛠️ 流程：`file_watcher.close()` → `file_store.clear()` → `file_watcher.update_store()` → `file_watcher.start()`（finally 保证重启） |
+| 🔎 search | 🔍 `search` (`search_step`) | `call_server("search", query=…, …)` | 📥 `query:str` ⭐ ｜ 🎚️ `limit:int=5`(>0) ｜ 🎚️ `min_score:float=0.0` ｜ ⚖️ `vector_weight:float=0.7` ∈[0,1]（keyword 权 = 1-vw）｜ 🔀 `candidate_multiplier:float=3.0`（candidates = min(200, limit×mult)）｜ 🔗 `expand_links:bool=True` ｜ 🔢 `max_links_per_direction:int=10` ｜ 🎚️ `search_filter:dict={}` ｜ 📤 `answer` 每命中一行 `path:start-end [score=… vector=… keyword=…] text` + 缩进的 `→ outlinks (n)` / `← inlinks (n)` + `via predicate=… anchor=#…` ｜ 📊 `metadata.results` / `metadata.link_expansion` / `metadata.counts={vector,keyword,returned,hybrid}` ｜ 🛠️ 并行 `vector_search` + `keyword_search` → RRF 融合（K=60，按 chunk.id 合并）→ `min_score` 过滤 → `limit` 截断 → 邻居 meta 注入 |
+| 🧪 demo | 🪄 `demo_echo` (`demo_echo_step1` + `step2`) | `call_server("demo_echo", query=…, min_score=…)` | 📥 `query:str=""` ｜ 🎚️ `min_score:float=0.5` ｜ 🛠️ step1：`processed_query = query.strip().lower()`，`adjusted_min_score = min_score * 0.9`，写回 context ｜ 📤 step2：`answer = "echo: {processed_query} (min_score={adjusted_min_score})"` ｜ 📊 `metadata = {step, query, min_score, processed_query, adjusted_min_score}` |
+| 🌊 demo | 🌊 `stream_demo` (`stream_demo_step1` + `step2`) | `call_server("stream_demo", query=…, repeat=…, interval=…)` | 📥 `query:str=""` ｜ 🎚️ `repeat:int=10` ｜ 🎚️ `interval:float=0.1`（秒/字符）｜ 🛠️ step1：`stream_text = query * repeat` 写回 context ｜ 📤 step2：按字符 `add_stream_string(ch, ChunkEnum.CONTENT)` 流式输出，`asyncio.sleep(interval)` 节流 |
+
+使用示例：
+
+```bash
+# 启动（默认 default.yaml）
+reme4 start
+
+# 指定 config 与服务端口
+reme4 start config=paw.yaml service.port=8181
+
+# 查找在跑的 reme
+reme4 find_reme
+# HOST=127.0.0.1 PORT=8000 PID=12345
+
+# 列出所有可用 action（client 端处理，不转服务端）
+reme4 list
+
+# 转发到服务端的 step：所有 key=value 透传为 step kwargs
+reme4 help
+reme4 health_check
+reme4 version
+reme4 reindex
+reme4 search query="latency 问题" limit=10 min_score=0.2 vector_weight=0.6
+
+# 通过 MCP backend 调用
+reme4 search query="..." backend=mcp
+```
@sen
| tags | stat | 返回特定tag信息 |
From 357415dd49eb24c52b316593c9bd0c3b30490b53 Mon Sep 17 00:00:00 2001
From: Sen Huang <48879559+ployts@users.noreply.github.com>
Date: Mon, 18 May 2026 14:25:14 +0800
Subject: [PATCH 11/16] feat: add Neo4j file graph support and markdown parser
with wikilink extraction (#240)
* feat: add Neo4j file graph support and markdown parser with wikilink extraction
- Add Neo4jFileGraph implementation for property-graph storage with
virtual/real node handling and link management
- Introduce LinkedFileParser for markdown files with frontmatter,
wikilink graph extraction, and full-skeleton chunking
- Update pyproject.toml to include pyyaml, mistletoe, and neo4j
dependencies
- Modify .gitignore to exclude /vault and structure.md
- Change reme CLI entry point from reme_ai.main to remecli.reme
- Register new neo4j and md components in respective registries
* refactor(file-graph): add chunk_ids support to Neo4jFileGraph
Add chunk_ids field to File node properties in Neo4jFileGraph to
enable better content chunk tracking and management.
BREAKING CHANGE: File node schema now includes chunk_ids property
which may affect existing integrations.
feat(parser): implement wikilink resolution logic
Move path resolution logic from utils/path_resolver to
linked_file_parser module and enhance wikilink resolution with
folder-note rule support and improved error handling.
fix(tests): update test assertions and variable names
Update test cases to reflect changes in data structures and
variable naming conventions across various components.
chore(config): update package entry point reference
Change reme CLI entry point from remecli.reme:main to
reme_ai.reme:main in pyproject.toml.
refactor(utils): remove deprecated path_resolver module
Remove the old path_resolver utility module as its functionality
has been moved to linked_file_parser.
docs(file-graph): update Neo4jFileGraph documentation
Update class docstrings and comments to reflect new chunk_ids
property and other structural changes.
style(formatting): adjust code formatting and line breaks
Minor formatting improvements including line length optimization
and consistent spacing adjustments throughout the codebase.
* fix(pyproject.toml): correct entry point for reme command
Change the entry point from "reme_ai.reme:main" to "reme_ai.main:main"
to fix the module reference for the reme command in project scripts.
---
.gitignore | 2 +-
pyproject.toml | 3 +
reme4/components/file_graph/__init__.py | 3 +-
.../components/file_graph/neo4j_file_graph.py | 450 +++++++++++
reme4/components/file_parser/__init__.py | 3 +-
.../file_parser/linked_file_parser.py | 729 ++++++++++++++++++
reme4/utils/logo_utils.py | 28 +-
tests4/unittest/test_linked_file_parser.py | 284 +++++++
tests4/unittest/test_neo4j_file_graph.py | 338 ++++++++
9 files changed, 1831 insertions(+), 9 deletions(-)
create mode 100644 reme4/components/file_graph/neo4j_file_graph.py
create mode 100644 reme4/components/file_parser/linked_file_parser.py
create mode 100644 tests4/unittest/test_linked_file_parser.py
create mode 100644 tests4/unittest/test_neo4j_file_graph.py
diff --git a/.gitignore b/.gitignore
index b20570b5..264c7372 100644
--- a/.gitignore
+++ b/.gitignore
@@ -42,4 +42,4 @@ meta_memory/*
**/data/*.json
*.db
memories/*
-.reme/*
\ No newline at end of file
+.reme/*
diff --git a/pyproject.toml b/pyproject.toml
index c9f644af..81039167 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -63,6 +63,9 @@ dependencies = [
"transformers>=4.57.3",
"uvicorn>=0.40.0",
"watchfiles>=1.1.1",
+ "pyyaml>=6.0.3",
+ "mistletoe",
+ "neo4j",
]
[project.optional-dependencies]
diff --git a/reme4/components/file_graph/__init__.py b/reme4/components/file_graph/__init__.py
index 12b01355..257bc7dc 100644
--- a/reme4/components/file_graph/__init__.py
+++ b/reme4/components/file_graph/__init__.py
@@ -2,6 +2,7 @@
from .base_file_graph import BaseFileGraph
from .local_file_graph import LocalFileGraph
+from .neo4j_file_graph import Neo4jFileGraph
from .nx_file_graph import NxFileGraph
-__all__ = ["BaseFileGraph", "LocalFileGraph", "NxFileGraph"]
+__all__ = ["BaseFileGraph", "LocalFileGraph", "Neo4jFileGraph", "NxFileGraph"]
diff --git a/reme4/components/file_graph/neo4j_file_graph.py b/reme4/components/file_graph/neo4j_file_graph.py
new file mode 100644
index 00000000..869c2b18
--- /dev/null
+++ b/reme4/components/file_graph/neo4j_file_graph.py
@@ -0,0 +1,450 @@
+"""Neo4j-backed file graph.
+
+Property-graph mapping:
+
+ Real node: (:File {path, st_mtime, title, description, tags,
+ chunk_ids, links_json, extra_json})
+ Virtual node: (:File {path}) — placeholder created when something
+ links to a path that hasn't been upserted yet.
+
+ Edge: (:File)-[:LINKS {idx, anchor, predicate}]->(:File)
+
+The ``links_json`` property doubles as the "is real" marker — its
+presence means the node was upserted with a payload; its absence
+means the node exists only because some edge points at it. This
+mirrors ``NxFileGraph`` exactly: ``upsert_nodes`` promotes virtuals
+in place, ``delete_nodes`` demotes back to virtual (or fully removes
+if nothing points here), and ``get_outlinks`` excludes edges into
+virtuals so the agent never sees dangling pointers.
+
+``path`` is the unique key (constraint enforced on ``_start``).
+Frontmatter goes into flat properties; arbitrary extras land in
+``extra_json``. The full ``FileLink[]`` payload is also stored as
+``links_json`` so ``rebuild_links`` can rebuild the relationship
+graph from per-node payloads after backend repair / migration.
+
+Adjacency policy: trusts ``FileLink.path`` directly — no internal
+wikilink resolution. The parser pipeline (with the external
+resolver) produces safe links where ``link.path`` is already a
+vault-relative target.
+
+Conditional dependency: the ``neo4j`` driver loads lazily; the
+import error fires at ``_start`` (boot), not at first call.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from .base_file_graph import BaseFileGraph
+from ..component_registry import R
+from ...schema import FileLink, FileNode
+from ...schema.file_node import FileFrontMatter
+
+
+_TYPED_FRONTMATTER_FIELDS = {"title", "description", "tags"}
+_LINK_FIELDS = {"source_path", "target_path", "target_anchor", "predicate"}
+
+# Properties that distinguish a "real" node from a virtual placeholder.
+# Listed for the demote query (delete_nodes) so we can REMOVE them all.
+_REAL_PROPS = (
+ "st_mtime",
+ "title",
+ "description",
+ "tags",
+ "chunk_ids",
+ "links_json",
+ "extra_json",
+)
+
+
+@R.register("neo4j")
+class Neo4jFileGraph(BaseFileGraph):
+ """Neo4j-backed file graph; trusts ``FileLink.path`` for adjacency.
+
+ Connection params (constructor kwargs):
+ uri: bolt URL, e.g. ``bolt://localhost:7687``
+ user: auth user (default ``neo4j``)
+ password: auth password
+ database: target db name (default ``neo4j``)
+ """
+
+ def __init__(
+ self,
+ uri: str = "bolt://localhost:7687",
+ user: str = "neo4j",
+ password: str = "neo4j",
+ database: str = "neo4j",
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ self._uri: str = uri
+ self._user: str = user
+ self._password: str = password
+ self._database: str = database
+ self._driver = None
+
+ # -- Lifecycle ---------------------------------------------------------
+
+ async def _start(self) -> None:
+ await super()._start()
+ try:
+ from neo4j import AsyncGraphDatabase
+ except ImportError as e:
+ raise ImportError(
+ "Neo4jFileGraph requires the neo4j driver. Install with `pip install neo4j`.",
+ ) from e
+ self._driver = AsyncGraphDatabase.driver(
+ self._uri,
+ auth=(self._user, self._password),
+ )
+ async with self._session() as session:
+ await session.run(
+ "CREATE CONSTRAINT file_path_unique IF NOT EXISTS FOR (f:File) REQUIRE f.path IS UNIQUE",
+ )
+ real, virtual, edges = await self._counts(session)
+ self.logger.info(
+ f"Neo4jFileGraph '{self.graph_name}' connected at "
+ f"{self._uri}/{self._database}: "
+ f"{real} nodes, {edges} edges, {virtual} virtual",
+ )
+
+ async def _close(self) -> None:
+ if self._driver is not None:
+ await self._driver.close()
+ self._driver = None
+ await super()._close()
+
+ def _session(self):
+ assert self._driver is not None, "Neo4jFileGraph not started"
+ return self._driver.session(database=self._database)
+
+ @staticmethod
+ async def _counts(session) -> tuple[int, int, int]:
+ rec = await session.run(
+ """
+ MATCH (f:File)
+ WITH count(CASE WHEN f.links_json IS NOT NULL THEN 1 END) AS real,
+ count(CASE WHEN f.links_json IS NULL THEN 1 END) AS virtual
+ OPTIONAL MATCH ()-[r:LINKS]->()
+ RETURN real, virtual, count(r) AS edges
+ """,
+ )
+ row = await rec.single()
+ if row is None:
+ return 0, 0, 0
+ return int(row["real"] or 0), int(row["virtual"] or 0), int(row["edges"] or 0)
+
+ # -- Node CRUD ---------------------------------------------------------
+
+ async def upsert_nodes(self, nodes: list[FileNode]) -> None:
+ """Upsert in one tx: SET props (promotes virtual to real), drop
+ existing outgoing edges, re-emit edges (auto-creating virtual
+ nodes for unindexed targets)."""
+ if not nodes:
+ return
+ payload = [
+ {
+ "path": node.path,
+ "props": self._node_props(node),
+ "links": [
+ {
+ "idx": i,
+ "anchor": link.target_anchor,
+ "predicate": link.predicate,
+ "target": link.target_path,
+ }
+ for i, link in enumerate(node.links)
+ if link.target_path
+ ],
+ }
+ for node in nodes
+ ]
+ async with self._session() as session:
+ await session.execute_write(self._upsert_nodes_tx, payload)
+
+ @staticmethod
+ async def _upsert_nodes_tx(tx, payload):
+ # 1. Upsert node props (promotes virtual → real where necessary).
+ await tx.run(
+ """
+ UNWIND $items AS n
+ MERGE (f:File {path: n.path})
+ SET f += n.props
+ """,
+ items=payload,
+ )
+ # 2. Drop existing outgoing edges from these sources.
+ await tx.run(
+ """
+ UNWIND $paths AS p
+ MATCH (f:File {path: p})-[r:LINKS]->()
+ DELETE r
+ """,
+ paths=[item["path"] for item in payload],
+ )
+ # 3. Re-emit edges; MERGE on target auto-creates virtual nodes
+ # for unindexed targets.
+ await tx.run(
+ """
+ UNWIND $items AS n
+ MATCH (s:File {path: n.path})
+ UNWIND n.links AS link
+ MERGE (t:File {path: link.target})
+ MERGE (s)-[r:LINKS {idx: link.idx}]->(t)
+ SET r.anchor = link.anchor, r.predicate = link.predicate
+ """,
+ items=payload,
+ )
+
+ async def delete_nodes(self, paths: list[str]) -> None:
+ """Demote real → virtual to preserve inbound visibility; fully
+ remove the (now-virtual) node only if no edge points at it."""
+ if not paths:
+ return
+ async with self._session() as session:
+ await session.execute_write(self._delete_nodes_tx, list(paths))
+
+ @staticmethod
+ async def _delete_nodes_tx(tx, paths):
+ # 1. Drop outgoing edges, then strip "real" properties (demote).
+ # Building the REMOVE clause from _REAL_PROPS keeps the list of
+ # properties in one place (top of module).
+ remove_clause = ", ".join(f"f.{name}" for name in _REAL_PROPS)
+ await tx.run(
+ f"""
+ UNWIND $paths AS p
+ MATCH (f:File {{path: p}})
+ OPTIONAL MATCH (f)-[r:LINKS]->()
+ DELETE r
+ WITH DISTINCT f
+ REMOVE {remove_clause}
+ """,
+ paths=paths,
+ )
+ # 2. Garbage-collect: drop the virtual node entirely if nothing
+ # points at it anymore.
+ await tx.run(
+ """
+ UNWIND $paths AS p
+ MATCH (f:File {path: p})
+ WHERE f.links_json IS NULL AND NOT (f)<-[:LINKS]-()
+ DELETE f
+ """,
+ paths=paths,
+ )
+
+ async def get_nodes(self, paths: list[str] | None = None) -> list[FileNode]:
+ """Return real nodes (virtual placeholders filtered).
+
+ ``paths=None`` streams every real node ordered by path. An
+ explicit ``[]`` returns ``[]`` without hitting the database.
+ """
+ if paths is not None and not paths:
+ return []
+ async with self._session() as session:
+ if paths is None:
+ rec = await session.run(
+ """
+ MATCH (f:File)
+ WHERE f.links_json IS NOT NULL
+ RETURN f
+ ORDER BY f.path ASC
+ """,
+ )
+ else:
+ rec = await session.run(
+ """
+ UNWIND $paths AS p
+ MATCH (f:File {path: p})
+ WHERE f.links_json IS NOT NULL
+ RETURN f
+ """,
+ paths=list(paths),
+ )
+ rows = [row["f"] async for row in rec]
+ return [self._row_to_node(row) for row in rows]
+
+ async def rebuild_links(self) -> None:
+ """Defensive full rebuild from each real node's ``links_json``.
+
+ Three steps in one tx: drop all LINKS edges; drop all virtual
+ nodes; re-emit edges from per-node link payloads (re-creating
+ virtual targets as needed). Useful after manual repair or
+ schema migration.
+ """
+ async with self._session() as session:
+ rec = await session.run(
+ """
+ MATCH (f:File)
+ WHERE f.links_json IS NOT NULL
+ RETURN f.path AS p, f.links_json AS l
+ """,
+ )
+ rows = [dict(r) async for r in rec]
+
+ payload: list[dict] = []
+ for row in rows:
+ try:
+ links = json.loads(row.get("l") or "[]")
+ except json.JSONDecodeError:
+ continue
+ items = [
+ {
+ "idx": i,
+ "anchor": link.get("target_anchor"),
+ "predicate": link.get("predicate"),
+ "target": link.get("target_path"),
+ }
+ for i, link in enumerate(links)
+ if isinstance(link, dict) and link.get("target_path")
+ ]
+ payload.append({"path": row["p"], "links": items})
+
+ async with self._session() as session:
+ await session.execute_write(self._rebuild_links_tx, payload)
+
+ @staticmethod
+ async def _rebuild_links_tx(tx, payload):
+ # 1. Wipe all edges and all virtual nodes.
+ await tx.run("MATCH ()-[r:LINKS]->() DELETE r")
+ await tx.run("MATCH (f:File) WHERE f.links_json IS NULL DELETE f")
+ if not payload:
+ return
+ # 2. Re-emit edges; virtual targets reappear via MERGE.
+ await tx.run(
+ """
+ UNWIND $items AS n
+ MATCH (s:File {path: n.path})
+ UNWIND n.links AS link
+ MERGE (t:File {path: link.target})
+ MERGE (s)-[r:LINKS {idx: link.idx}]->(t)
+ SET r.anchor = link.anchor, r.predicate = link.predicate
+ """,
+ items=payload,
+ )
+
+ async def clear(self):
+ """Remove every node and edge in the configured database."""
+ async with self._session() as session:
+ await session.run("MATCH (f:File) DETACH DELETE f")
+
+ # -- Link access -------------------------------------------------------
+
+ async def get_outlinks(self, path: str) -> list[FileLink]:
+ """Outgoing links from ``path``. Source must be real; targets
+ into virtual nodes are excluded so dangling refs are invisible."""
+ async with self._session() as session:
+ rec = await session.run(
+ """
+ MATCH (s:File {path: $path})
+ WHERE s.links_json IS NOT NULL
+ MATCH (s)-[r:LINKS]->(t:File)
+ WHERE t.links_json IS NOT NULL
+ RETURN t.path AS target, r.anchor AS anchor,
+ r.predicate AS predicate, r.idx AS idx
+ ORDER BY r.idx ASC
+ """,
+ path=path,
+ )
+ rows = [dict(row) async for row in rec]
+ return [
+ FileLink(
+ source_path=path,
+ target_path=row["target"],
+ target_anchor=row.get("anchor"),
+ predicate=row.get("predicate"),
+ )
+ for row in rows
+ ]
+
+ async def get_inlinks(self, path: str) -> list[FileLink]:
+ """Incoming links to ``path`` (must be real). Sources are always
+ real because virtual nodes never have outgoing edges."""
+ async with self._session() as session:
+ rec = await session.run(
+ """
+ MATCH (t:File {path: $path})
+ WHERE t.links_json IS NOT NULL
+ MATCH (s:File)-[r:LINKS]->(t)
+ RETURN r.anchor AS anchor, r.predicate AS predicate,
+ r.idx AS idx, s.path AS source
+ ORDER BY s.path ASC, r.idx ASC
+ """,
+ path=path,
+ )
+ rows = [dict(row) async for row in rec]
+ return [
+ FileLink(
+ source_path=row["source"],
+ target_path=path,
+ target_anchor=row.get("anchor"),
+ predicate=row.get("predicate"),
+ )
+ for row in rows
+ ]
+
+ # -- Internal: row ↔ schema marshaling ---------------------------------
+
+ @staticmethod
+ def _node_props(node: FileNode) -> dict[str, Any]:
+ fm = node.front_matter
+ extras = dict(fm.__pydantic_extra__ or {})
+ return {
+ "path": node.path,
+ "st_mtime": float(node.st_mtime),
+ "title": fm.title or "",
+ "description": fm.description or "",
+ "tags": list(fm.tags or []),
+ "chunk_ids": list(node.chunk_ids or []),
+ "links_json": json.dumps(
+ [link.model_dump(exclude_none=True) for link in node.links],
+ ensure_ascii=False,
+ ),
+ "extra_json": json.dumps(extras, ensure_ascii=False, sort_keys=True),
+ }
+
+ @staticmethod
+ def _row_to_node(row) -> FileNode:
+ d = dict(row)
+ try:
+ extras = json.loads(d.get("extra_json") or "{}")
+ except json.JSONDecodeError:
+ extras = {}
+ try:
+ links_raw = json.loads(d.get("links_json") or "[]")
+ except json.JSONDecodeError:
+ links_raw = []
+ links: list[FileLink] = []
+ for link in links_raw:
+ if not isinstance(link, dict):
+ continue
+ # Defensive: strip any keys the schema doesn't recognise
+ # (e.g. legacy fields from prior schema versions).
+ clean = {k: v for k, v in link.items() if k in _LINK_FIELDS}
+ # Ensure source_path is populated — older payloads (or
+ # links written before the schema split) only carry the
+ # target side; default to the owning node's path.
+ clean.setdefault("source_path", d["path"])
+ if not clean.get("target_path"):
+ continue
+ try:
+ links.append(FileLink(**clean))
+ except Exception:
+ continue
+ fm_kwargs: dict[str, Any] = {
+ "title": d.get("title", "") or "",
+ "description": d.get("description", "") or "",
+ "tags": d.get("tags") or None,
+ }
+ fm_kwargs.update(
+ {k: v for k, v in extras.items() if k not in _TYPED_FRONTMATTER_FIELDS},
+ )
+ return FileNode(
+ path=d["path"],
+ st_mtime=float(d.get("st_mtime", 0.0)),
+ links=links,
+ chunk_ids=[str(c) for c in (d.get("chunk_ids") or [])],
+ front_matter=FileFrontMatter(**fm_kwargs),
+ )
diff --git a/reme4/components/file_parser/__init__.py b/reme4/components/file_parser/__init__.py
index 3e20d0bf..c5b62bd9 100644
--- a/reme4/components/file_parser/__init__.py
+++ b/reme4/components/file_parser/__init__.py
@@ -3,5 +3,6 @@
from .bare_file_parser import BareFileParser
from .base_file_parser import BaseFileParser
from .default_file_parser import DefaultFileParser
+from .linked_file_parser import LinkedFileParser
-__all__ = ["BareFileParser", "BaseFileParser", "DefaultFileParser"]
+__all__ = ["BareFileParser", "BaseFileParser", "DefaultFileParser", "LinkedFileParser"]
diff --git a/reme4/components/file_parser/linked_file_parser.py b/reme4/components/file_parser/linked_file_parser.py
new file mode 100644
index 00000000..2c825dfa
--- /dev/null
+++ b/reme4/components/file_parser/linked_file_parser.py
@@ -0,0 +1,729 @@
+"""Markdown file parser — frontmatter + wikilink graph + AST tree chunks.
+
+Each chunk carries the **complete heading skeleton** of the document
+with its content inlined under the section that owns it; other sections
+appear as bare headings so the reader always sees a full document map.
+
+Pipeline: build mistletoe AST → ``MdNode`` tree (sections nest by
+heading level) → recursive chunk (try whole subtree; on overflow walk
+children — body siblings pack as a run, subsections recurse). Leaf
+blocks (table / code / list / paragraph) split on internal boundaries
+and each piece is annotated ``[Part X/N]``. Wikilinks in the body are
+extracted as graph edges, with optional Dataview-style typed predicates
+(line-level ``predicate:: [[X]]`` or inline-bracketed ``[predicate:: [[X]]]``).
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import frontmatter
+
+
+from .base_file_parser import BaseFileParser
+from ..component_registry import R
+from ..file_graph import BaseFileGraph
+from ...enumeration import ComponentEnum
+from ...schema import (
+ FileChunk,
+ FileLink,
+ FileFrontMatter,
+ FileNode,
+)
+
+
+# -- Wikilink resolution --------------------------------------------------
+#
+# Wikilinks are a markdown user-facing convention: ``[[Alice]]`` should
+# resolve to ``topics/Alice/Alice.md`` (or wherever the file lives).
+# This short-form / implicit-``.md`` / folder-note resolution lives here
+# at the markdown boundary rather than as a generic utility — file-IO
+# steps require full vault-relative paths and never use these helpers.
+
+
+def _complete_md(target: str) -> str:
+ """Apply implicit ``.md`` rule for wikilink targets."""
+ if not target:
+ return target
+ last = target.rsplit("/", 1)[-1]
+ return target if "." in last else target + ".md"
+
+
+def _filter_folder_note(target: str, paths: list[str]) -> list[str]:
+ """Apply folder-note rule: when both ``X.md`` and ``X/X.md`` exist,
+ prefer ``X/X.md``. Sorted for determinism.
+ """
+ if not paths:
+ return []
+ stem = Path(target).stem
+ folder_hits = sorted(p for p in paths if Path(p).parent.name == stem)
+ return folder_hits or sorted(paths)
+
+
+async def _resolve_wikilink(graph: BaseFileGraph, target: str) -> list[str]:
+ """Resolve a wikilink target to vault-relative path(s).
+
+ Returns:
+ ``[path]`` for an unambiguous match,
+ ``[path, path, ...]`` for short-form ambiguity (caller may
+ fan out one FileLink per candidate), or
+ ``[]`` when nothing matches (dangling — caller drops the link).
+ """
+ if not target:
+ return []
+ target = _complete_md(target)
+ if "/" in target:
+ nodes = await graph.get_nodes([target])
+ return [target] if nodes else []
+ matches = [n.path for n in await graph.get_nodes() if Path(n.path).name == target]
+ return _filter_folder_note(target, matches)
+
+
+# -- Wikilink extraction --------------------------------------------------
+
+
+_WIKILINK_RE = re.compile(
+ r"""
+ (?:!)?
+ \[\[
+ (?P[^\]\|\#\n]+?)
+ (?:\#(?P[^\]\|\n]+))?
+ (?:\|[^\]\n]+)?
+ \]\]
+ """,
+ re.VERBOSE,
+)
+
+_DATAVIEW_LINE_RE = re.compile(
+ r"^[ \t]*(?:[-*+][ \t]+)?(?P[A-Za-z][A-Za-z0-9_]*)\s*::\s*(?P.+?)\s*$",
+ re.MULTILINE,
+)
+
+_INLINE_FIELD_OPEN_RE = re.compile(r"\[(?P[A-Za-z][A-Za-z0-9_]*)\s*::\s*")
+
+
+def _iter_inline_fields(text: str) -> list[tuple[int, int, str]]:
+ """Find inline-bracketed ``[predicate:: …]`` field spans by depth scan."""
+ out: list[tuple[int, int, str]] = []
+ for m in _INLINE_FIELD_OPEN_RE.finditer(text):
+ depth = 1
+ i = m.end()
+ n = len(text)
+ while i < n:
+ c = text[i]
+ if c == "\n":
+ break
+ if c == "[":
+ depth += 1
+ elif c == "]":
+ depth -= 1
+ if depth == 0:
+ out.append((m.start(), i + 1, m.group("predicate")))
+ break
+ i += 1
+ return out
+
+
+def _predicate_for(
+ text: str,
+ pos: int,
+ inline_spans: list[tuple[int, int, str]],
+) -> str | None:
+ """Resolve the predicate governing a wikilink at offset ``pos``.
+
+ Precedence: inline-bracketed > line-level Dataview > none.
+ """
+ for field_start, field_end, predicate in inline_spans:
+ if field_start <= pos < field_end:
+ return predicate
+ line_start = text.rfind("\n", 0, pos) + 1
+ line_end = text.find("\n", pos)
+ if line_end == -1:
+ line_end = len(text)
+ m = _DATAVIEW_LINE_RE.match(text[line_start:line_end])
+ if m and line_start + m.start("value") <= pos:
+ return m.group("predicate")
+ return None
+
+
+async def _extract_links(
+ graph: BaseFileGraph,
+ text: str,
+ source_path: str,
+) -> list[FileLink]:
+ """Find every wikilink in ``text``, resolve targets, emit FileLinks.
+
+ Short-path ambiguity **expands** into one FileLink per candidate so
+ the body's wikilink is recorded against every plausible target.
+ Dangling targets are dropped. Results are deduped by
+ ``(target_path, predicate, target_anchor)`` preserving order.
+ """
+ if not text:
+ return []
+ inline_spans = _iter_inline_fields(text)
+ out: list[FileLink] = []
+ seen: set[tuple] = set()
+ for wm in _WIKILINK_RE.finditer(text):
+ target = wm.group("target").strip()
+ if not target:
+ continue
+ anchor_raw = wm.group("anchor")
+ anchor = anchor_raw.strip() if anchor_raw else None
+ predicate = _predicate_for(text, wm.start(), inline_spans)
+ resolved_paths = await _resolve_wikilink(graph, target)
+ if not resolved_paths:
+ continue
+ for resolved in resolved_paths:
+ key = (resolved, predicate, anchor)
+ if key in seen:
+ continue
+ seen.add(key)
+ out.append(
+ FileLink(
+ source_path=source_path,
+ target_path=resolved,
+ target_anchor=anchor,
+ predicate=predicate,
+ ),
+ )
+ return out
+
+
+# -- AST node + helpers ---------------------------------------------------
+
+
+@dataclass
+class MdNode:
+ """``root`` / ``section`` (heading + children until equal-or-shallower
+ heading) / ``body`` (one mistletoe block; ``block`` keeps the original).
+
+ ``text`` is the rendered subtree (own heading excluded for sections).
+ ``desc_toc`` caches the section-only DFS outline of descendants
+ (own heading excluded), used as the TOC suffix when emitting chunks
+ inside a section. Line ranges span the full subtree.
+ """
+
+ kind: str # "root" | "section" | "body"
+ heading: str | None = None
+ level: int = 0
+ children: list["MdNode"] = field(default_factory=list)
+ block: Any = None
+ text: str = ""
+ start_line: int = 0
+ end_line: int = 0
+ desc_toc: str = ""
+
+
+def _heading_text(node: Any, renderer) -> str:
+ """Heading text without `#` markers (for outline)."""
+ rendered = renderer.render(node).rstrip("\n")
+ if rendered.startswith("#"):
+ return rendered.lstrip("#").strip()
+ return rendered.split("\n", 1)[0].strip()
+
+
+def _finalize(n: MdNode) -> None:
+ """Bottom-up pass: propagate line ranges, populate ``n.text`` (rendered
+ subtree, own heading excluded for sections) and ``n.desc_toc`` (DFS
+ section outline of descendants)."""
+ parts: list[str] = []
+ desc_lines: list[str] = []
+ for c in n.children:
+ _finalize(c)
+ if c.kind == "section":
+ heading = f"{'#' * c.level} {c.heading or ''}"
+ parts.append(f"{heading}\n\n{c.text}" if c.text else heading)
+ desc_lines.append(f"{heading}\n\n{c.desc_toc}" if c.desc_toc else heading)
+ elif c.text:
+ parts.append(c.text)
+ if n.children:
+ first = n.children[0].start_line
+ n.start_line = min(n.start_line, first) if n.start_line else first
+ n.end_line = max(c.end_line for c in n.children)
+ elif n.end_line < n.start_line:
+ n.end_line = n.start_line
+ if n.kind != "body":
+ n.text = "\n\n".join(parts)
+ n.desc_toc = "\n\n".join(desc_lines)
+
+
+def _toc_join(*parts: str) -> str:
+ """Concatenate TOC fragments with ``\\n\\n``, skipping empty ones."""
+ return "\n\n".join(p for p in parts if p)
+
+
+def _subtree_toc(n: MdNode) -> str:
+ """Section's heading + descendants TOC — its contribution to a parent's
+ ``desc_toc``. For root (no own heading) this is just ``desc_toc``."""
+ if n.kind != "section" or n.heading is None:
+ return n.desc_toc
+ heading = f"{'#' * n.level} {n.heading}"
+ return f"{heading}\n\n{n.desc_toc}" if n.desc_toc else heading
+
+
+# -- Parser ---------------------------------------------------------------
+
+
+@R.register("md")
+class LinkedFileParser(BaseFileParser):
+ """Markdown parser: frontmatter + wikilink edges + full-skeleton chunks."""
+
+ def __init__(
+ self,
+ encoding: str = "utf-8",
+ chunk_chars: int = 2000,
+ embed_toc: bool = True,
+ file_graph: str = "default",
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ self.encoding = encoding
+ self.chunk_chars = max(100, chunk_chars)
+ self.embed_toc = embed_toc
+ self._file_graph_name: str = file_graph
+
+ def _resolve_file_graph(self) -> BaseFileGraph | None:
+ """Lazily fetch the configured file_graph from app_context.
+
+ Lazy (rather than ``_start``) so the parser doesn't impose a
+ component start-order constraint, and so tests can construct
+ the parser without a graph wired up.
+ """
+ if self.app_context is None:
+ return None
+ graphs = self.app_context.components.get(ComponentEnum.FILE_GRAPH, {})
+ graph = graphs.get(self._file_graph_name)
+ if graph is None:
+ return None
+ if not isinstance(graph, BaseFileGraph):
+ raise TypeError(
+ f"Expected BaseFileGraph, got {type(graph).__name__}",
+ )
+ return graph
+
+ async def parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]:
+ from mistletoe.markdown_renderer import MarkdownRenderer
+ from mistletoe.block_token import Document
+
+ file_path = Path(path)
+ rel_path = self._get_relative_path(path)
+ post = frontmatter.loads(file_path.read_text(encoding=self.encoding))
+
+ chunks: list[FileChunk] = []
+ if post.content and post.content.strip():
+ with MarkdownRenderer() as renderer:
+ tree = self._build_tree(Document(post.content), renderer)
+ chunks = self._chunk_node(tree, "", "", rel_path, renderer)
+
+ links: list[FileLink] = []
+ graph = self._resolve_file_graph()
+ if graph is not None:
+ links = await _extract_links(graph, post.content, rel_path)
+
+ node = FileNode(
+ path=rel_path,
+ st_mtime=file_path.stat().st_mtime,
+ chunk_ids=[chunk.id for chunk in chunks],
+ links=links,
+ front_matter=FileFrontMatter(**dict(post.metadata)),
+ )
+ return node, chunks
+
+ def _build_tree(self, doc: Any, renderer) -> MdNode:
+ """Heading-level stack folds mistletoe's flat children into nested
+ sections; non-headings attach as ``body`` to the current section
+ (or root before the first heading)."""
+ from mistletoe.markdown_renderer import BlankLine
+ from mistletoe.block_token import (
+ Heading,
+ SetextHeading,
+ )
+
+ root = MdNode(kind="root", start_line=1, end_line=1)
+ stack: list[MdNode] = [root]
+ for child in doc.children or []:
+ if isinstance(child, BlankLine):
+ continue
+ line = getattr(child, "line_number", None) or stack[-1].start_line
+ if isinstance(child, (Heading, SetextHeading)):
+ level = max(1, getattr(child, "level", 1))
+ while len(stack) > 1 and stack[-1].level >= level:
+ stack.pop()
+ sec = MdNode(
+ kind="section",
+ heading=_heading_text(child, renderer),
+ level=level,
+ start_line=line,
+ )
+ stack[-1].children.append(sec)
+ stack.append(sec)
+ continue
+ rendered = renderer.render(child).rstrip("\n")
+ if not rendered:
+ continue
+ stack[-1].children.append(
+ MdNode(
+ kind="body",
+ block=child,
+ text=rendered,
+ start_line=line,
+ end_line=line + rendered.count("\n"),
+ ),
+ )
+ _finalize(root)
+ return root
+
+ # -- Recursive chunker ------------------------------------------------
+
+ def _chunk_node(
+ self,
+ node: MdNode,
+ before: str,
+ after: str,
+ path: str,
+ renderer,
+ ) -> list[FileChunk]:
+ """Try the whole subtree; on overflow split (leaf) or descend.
+ ``before``/``after`` are TOC fragments that bracket each emitted
+ chunk's content (chunk text = ``before + content + after``).
+ As we descend, the prefix grows with section headings already
+ passed and the suffix shrinks correspondingly.
+ """
+ if not node.text:
+ return []
+ if node.kind == "section":
+ heading_line = f"{'#' * node.level} {node.heading or ''}"
+ before_self = _toc_join(before, heading_line)
+ else:
+ before_self = before
+ if len(node.text) <= self.chunk_chars:
+ return [
+ self._make_chunk(
+ before_self,
+ node.text,
+ after,
+ node.start_line,
+ node.end_line,
+ path,
+ ),
+ ]
+ if node.kind == "body":
+ return self._split_leaf(node, before, after, path, renderer)
+ after_inside = _toc_join(node.desc_toc, after)
+ sub_tocs = [_subtree_toc(c) for c in node.children if c.kind == "section"]
+ chunks: list[FileChunk] = []
+ accumulated = before_self
+ sec_idx = 0
+ run: list[MdNode] = []
+ for c in node.children:
+ if c.kind == "section":
+ if run:
+ chunks.extend(
+ self._chunk_body_run(
+ run,
+ before_self,
+ after_inside,
+ path,
+ renderer,
+ ),
+ )
+ run = []
+ remaining = "\n\n".join(sub_tocs[sec_idx + 1 :])
+ chunks.extend(
+ self._chunk_node(
+ c,
+ accumulated,
+ _toc_join(remaining, after),
+ path,
+ renderer,
+ ),
+ )
+ accumulated = _toc_join(accumulated, sub_tocs[sec_idx])
+ sec_idx += 1
+ else:
+ run.append(c)
+ if run:
+ chunks.extend(
+ self._chunk_body_run(
+ run,
+ before_self,
+ after_inside,
+ path,
+ renderer,
+ ),
+ )
+ return chunks
+
+ def _chunk_body_run(
+ self,
+ run: list[MdNode],
+ before: str,
+ after: str,
+ path: str,
+ renderer,
+ ) -> list[FileChunk]:
+ """Greedy-pack consecutive body siblings under the same TOC slot.
+ No ``[Part X/N]`` markers — distinct blocks, not a leaf split.
+ Oversized single body recurses to ``_split_leaf``."""
+ composite_size = sum(len(b.text) for b in run) + 2 * max(0, len(run) - 1)
+ if composite_size <= self.chunk_chars:
+ return [
+ self._make_chunk(
+ before,
+ "\n\n".join(b.text for b in run),
+ after,
+ run[0].start_line,
+ run[-1].end_line,
+ path,
+ ),
+ ]
+
+ chunks: list[FileChunk] = []
+ bucket: list[MdNode] = []
+ bucket_chars = 0
+
+ def flush() -> None:
+ nonlocal bucket, bucket_chars
+ if not bucket:
+ return
+ chunks.append(
+ self._make_chunk(
+ before,
+ "\n\n".join(b.text for b in bucket),
+ after,
+ bucket[0].start_line,
+ bucket[-1].end_line,
+ path,
+ ),
+ )
+ bucket = []
+ bucket_chars = 0
+
+ for body in run:
+ if len(body.text) > self.chunk_chars:
+ flush()
+ chunks.extend(self._split_leaf(body, before, after, path, renderer))
+ continue
+ sep = 2 if bucket else 0
+ if bucket_chars + sep + len(body.text) > self.chunk_chars:
+ flush()
+ sep = 0
+ bucket.append(body)
+ bucket_chars += sep + len(body.text)
+ flush()
+ return chunks
+
+ # -- Leaf splitters: build (text, start, end) units, hand off to packer
+
+ def _split_leaf(
+ self,
+ body: MdNode,
+ before: str,
+ after: str,
+ path: str,
+ renderer,
+ ) -> list[FileChunk]:
+ from mistletoe.block_token import (
+ CodeFence,
+ List,
+ Table,
+ )
+
+ block = body.block
+ if isinstance(block, Table):
+ return self._split_table(body, before, after, path)
+ if isinstance(block, CodeFence):
+ return self._split_code(body, before, after, path)
+ if isinstance(block, List):
+ return self._split_list(body, before, after, path, renderer)
+ return self._split_lines(body, before, after, path)
+
+ def _split_table(
+ self,
+ body: MdNode,
+ before: str,
+ after: str,
+ path: str,
+ ) -> list[FileChunk]:
+ """Repeat header + separator on every chunk."""
+ from mistletoe.block_token import TableRow
+
+ lines = body.text.split("\n")
+ header, data = "\n".join(lines[:2]), lines[2:]
+ rows = [r for r in (body.block.children or []) if isinstance(r, TableRow)]
+ base = body.start_line + 2
+
+ def line_of(i: int) -> int:
+ return rows[i].line_number if i < len(rows) and rows[i].line_number else base + i
+
+ units = [(text, line_of(i), line_of(i)) for i, text in enumerate(data)]
+ return self._emit_packed(
+ units,
+ before,
+ after,
+ path,
+ joiner="\n",
+ wrap=f"{header}\n{{inner}}",
+ )
+
+ def _split_code(
+ self,
+ body: MdNode,
+ before: str,
+ after: str,
+ path: str,
+ ) -> list[FileChunk]:
+ """Repeat fence opener + closer on every chunk."""
+ code = body.block
+ indent = " " * (code.indentation or 0)
+ fence = f"{indent}{code.delimiter}"
+ opener = f"{fence}{code.info_string or ''}"
+ raw = (code.children[0].content if code.children else "").rstrip("\n")
+ if not raw:
+ return []
+ start = body.start_line + 1
+ units = [(indent + ln, start + i, start + i) for i, ln in enumerate(raw.split("\n"))]
+ return self._emit_packed(
+ units,
+ before,
+ after,
+ path,
+ joiner="\n",
+ wrap=f"{opener}\n{{inner}}\n{fence}",
+ allow_empty=True,
+ )
+
+ def _split_list(
+ self,
+ body: MdNode,
+ before: str,
+ after: str,
+ path: str,
+ renderer,
+ ) -> list[FileChunk]:
+ """Pack list items; oversized items emit alone (overflow accepted)."""
+ from mistletoe.block_token import ListItem
+
+ items = [c for c in (body.block.children or []) if isinstance(c, ListItem)]
+ if not items:
+ return self._split_lines(body, before, after, path)
+ units: list[tuple[str, int, int]] = []
+ for it in items:
+ text = renderer.render(it).rstrip("\n")
+ if not text:
+ continue
+ line = it.line_number or body.start_line
+ units.append((text, line, line + text.count("\n")))
+ return self._emit_packed(
+ units,
+ before,
+ after,
+ path,
+ joiner="\n",
+ wrap="{inner}",
+ )
+
+ def _split_lines(
+ self,
+ body: MdNode,
+ before: str,
+ after: str,
+ path: str,
+ ) -> list[FileChunk]:
+ """Last-resort line-greedy split for paragraphs / quotes / html."""
+ start = body.start_line
+ units = [(line, start + i, start + i) for i, line in enumerate(body.text.split("\n"))]
+ return self._emit_packed(
+ units,
+ before,
+ after,
+ path,
+ joiner="\n",
+ wrap="{inner}",
+ )
+
+ def _emit_packed(
+ self,
+ units: list[tuple[str, int, int]],
+ before: str,
+ after: str,
+ path: str,
+ joiner: str,
+ wrap: str,
+ allow_empty: bool = False,
+ ) -> list[FileChunk]:
+ """Greedy-pack units into ``wrap`` envelopes; emit each piece.
+
+ Envelope (table header, code fence) counts against ``chunk_chars``;
+ TOC (when on) is additive prefix/suffix downstream. Oversized
+ units overflow rather than truncate. Multi-piece outputs get
+ ``[Part X/N]`` markers; single pieces don't.
+ """
+ envelope = len(wrap.replace("{inner}", ""))
+ budget = max(64, self.chunk_chars - envelope)
+ sep_len = len(joiner)
+
+ parts: list[tuple[str, int, int]] = []
+ bucket: list[tuple[str, int, int]] = []
+ bucket_chars = 0
+
+ def flush() -> None:
+ nonlocal bucket, bucket_chars
+ if not bucket:
+ return
+ inner = joiner.join(t for t, _, _ in bucket)
+ parts.append((inner, bucket[0][1], bucket[-1][2]))
+ bucket = []
+ bucket_chars = 0
+
+ for text, s, e in units:
+ if not text and not allow_empty:
+ continue
+ sep = sep_len if bucket else 0
+ if bucket_chars + sep + len(text) > budget:
+ flush()
+ sep = 0
+ bucket.append((text, s, e))
+ bucket_chars += sep + len(text)
+ flush()
+
+ total = len(parts)
+ return [
+ self._make_chunk(
+ before,
+ (
+ f"[Part {idx}/{total}]\n\n{wrap.replace('{inner}', inner)}"
+ if total > 1
+ else wrap.replace("{inner}", inner)
+ ),
+ after,
+ s,
+ e,
+ path,
+ )
+ for idx, (inner, s, e) in enumerate(parts, 1)
+ ]
+
+ # -- Emit -------------------------------------------------------------
+
+ def _make_chunk(
+ self,
+ before: str,
+ content: str,
+ after: str,
+ start_line: int,
+ end_line: int,
+ path: str,
+ ) -> FileChunk:
+ """Build one ``FileChunk`` — text is ``before + content + after``
+ when ``embed_toc``, otherwise just ``content``."""
+ text = _toc_join(before, content, after) if self.embed_toc else content
+ return FileChunk(
+ path=path,
+ start_line=start_line,
+ end_line=end_line,
+ text=text,
+ ).set_hash_id()
diff --git a/reme4/utils/logo_utils.py b/reme4/utils/logo_utils.py
index 2a70ab94..b4795330 100644
--- a/reme4/utils/logo_utils.py
+++ b/reme4/utils/logo_utils.py
@@ -1,6 +1,8 @@
"""Startup banner with ASCII logo and service metadata."""
+import colorsys
import importlib.metadata
+import random
from typing import TYPE_CHECKING
from rich.console import Console, Group
@@ -20,8 +22,20 @@ def get_version(package_name: str) -> str:
return ""
+def _hsv_rgb(h: float, s: float = 0.85, v: float = 0.98) -> tuple[int, int, int]:
+ """HSV → 0-255 RGB tuple. High saturation+value keeps colors vibrant."""
+ r, g, b = colorsys.hsv_to_rgb(h % 1.0, s, v)
+ return int(r * 255), int(g * 255), int(b * 255)
+
+
def print_logo(app_config: "ApplicationConfig"):
- """Print gradient ASCII logo and runtime config (backend, URL, versions)."""
+ """Print rainbow ASCII logo and runtime config (backend, URL, versions).
+
+ Color: each startup picks a random hue rotation; both horizontal
+ (across each line) and vertical (line-to-line) sweep ~half the
+ hue wheel, so the banner shows a fresh multi-color rainbow gradient
+ every run.
+ """
ascii_art = [
r" ██████╗ ███████╗ ███╗ ███╗ ███████╗ ",
r" ██╔══██╗ ██╔════╝ ████╗ ████║ ██╔════╝ ",
@@ -31,16 +45,18 @@ def print_logo(app_config: "ApplicationConfig"):
r" ╚═╝ ╚═╝ ╚══════╝ ╚═╝ ╚═╝ ╚══════╝ ",
]
- start_color = (85, 239, 196)
- end_color = (162, 155, 254)
+ hue_base = random.random() # random starting hue per startup
+ horizontal_span = 0.5 # half the wheel left-to-right
+ vertical_shift = 0.08 # small per-line nudge for 2D rainbow
logo_text = Text()
- for line in ascii_art:
+ for line_idx, line in enumerate(ascii_art):
line_len = max(1, len(line) - 1)
+ line_hue_start = hue_base + line_idx * vertical_shift
for i, char in enumerate(line):
ratio = i / line_len
- rgb = tuple(int(s + (e - s) * ratio) for s, e in zip(start_color, end_color))
- logo_text.append(char, style=f"bold rgb({rgb[0]},{rgb[1]},{rgb[2]})")
+ r, g, b = _hsv_rgb(line_hue_start + horizontal_span * ratio)
+ logo_text.append(char, style=f"bold rgb({r},{g},{b})")
logo_text.append("\n")
info_table = Table.grid(padding=(0, 1))
diff --git a/tests4/unittest/test_linked_file_parser.py b/tests4/unittest/test_linked_file_parser.py
new file mode 100644
index 00000000..034d19ff
--- /dev/null
+++ b/tests4/unittest/test_linked_file_parser.py
@@ -0,0 +1,284 @@
+"""Tests for LinkedFileParser (markdown parser + wikilink extraction)."""
+
+# pylint: disable=protected-access
+
+import asyncio
+import os
+import tempfile
+
+from reme4.components.file_graph import LocalFileGraph
+from reme4.components.file_parser import LinkedFileParser
+from reme4.schema import FileNode
+
+
+class temp_chdir:
+ """Context manager to temporarily chdir into a path and restore on exit."""
+
+ def __init__(self, path):
+ self.path = path
+ self.old = None
+
+ def __enter__(self):
+ self.old = os.getcwd()
+ os.chdir(self.path)
+ return self
+
+ def __exit__(self, *exc):
+ os.chdir(self.old)
+
+
+def _write_md(tmpdir: str, name: str, body: str) -> str:
+ """Drop a markdown file under tmpdir, return its path."""
+ path = os.path.join(tmpdir, name)
+ if "/" in name:
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, "w", encoding="utf-8") as f:
+ f.write(body)
+ return path
+
+
+async def _make_graph(*nodes: FileNode) -> LocalFileGraph:
+ """Build a started LocalFileGraph seeded with the given nodes."""
+ graph = LocalFileGraph()
+ await graph.start()
+ if nodes:
+ await graph.upsert_nodes(list(nodes))
+ return graph
+
+
+def test_parse_empty_file():
+ """An empty .md → FileNode, no chunks, no links."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ path = _write_md(tmp, "empty.md", "")
+ parser = LinkedFileParser()
+ node, chunks = await parser.parse(path)
+ assert chunks == []
+ assert node.links == []
+ assert node.chunk_ids == []
+ print("✓ test_parse_empty_file passed")
+
+ asyncio.run(run())
+
+
+def test_parse_frontmatter_only():
+ """Front-matter without body → FileNode with metadata, no chunks/links."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ path = _write_md(tmp, "fm.md", "---\ntitle: Demo\ntags: [a, b]\n---\n")
+ parser = LinkedFileParser()
+ node, chunks = await parser.parse(path)
+ assert chunks == []
+ assert node.front_matter.title == "Demo"
+ assert list(node.front_matter.tags or []) == ["a", "b"]
+ print("✓ test_parse_frontmatter_only passed")
+
+ asyncio.run(run())
+
+
+def test_parse_small_body_one_chunk():
+ """A body shorter than chunk_chars produces exactly one chunk that contains the body."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ path = _write_md(tmp, "small.md", "# Hello\n\nworld")
+ parser = LinkedFileParser(chunk_chars=2000)
+ _, chunks = await parser.parse(path)
+ assert len(chunks) == 1
+ assert "world" in chunks[0].text
+ assert chunks[0].start_line >= 1
+ assert chunks[0].end_line >= chunks[0].start_line
+ print("✓ test_parse_small_body_one_chunk passed")
+
+ asyncio.run(run())
+
+
+def test_parse_oversized_body_splits():
+ """A body that exceeds chunk_chars produces multiple chunks."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ # 30 paragraphs of 100 chars each ≈ 3000 chars body
+ body = "# Big\n\n" + "\n\n".join("p" * 100 for _ in range(30))
+ path = _write_md(tmp, "big.md", body)
+ parser = LinkedFileParser(chunk_chars=500, embed_toc=False)
+ _, chunks = await parser.parse(path)
+ assert len(chunks) > 1
+ print("✓ test_parse_oversized_body_splits passed")
+
+ asyncio.run(run())
+
+
+def test_parse_chunk_ids_match_node_chunk_ids():
+ """FileNode.chunk_ids should match the ids of the chunks returned."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ path = _write_md(tmp, "ids.md", "# X\n\nbody")
+ parser = LinkedFileParser()
+ node, chunks = await parser.parse(path)
+ assert node.chunk_ids == [c.id for c in chunks]
+ print("✓ test_parse_chunk_ids_match_node_chunk_ids passed")
+
+ asyncio.run(run())
+
+
+def test_parse_links_empty_when_no_graph():
+ """Without an app_context / graph, links stay empty even if the body has wikilinks."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ path = _write_md(tmp, "x.md", "see [[Alice]] and [[Bob]]")
+ parser = LinkedFileParser()
+ node, _ = await parser.parse(path)
+ assert node.links == []
+ print("✓ test_parse_links_empty_when_no_graph passed")
+
+ asyncio.run(run())
+
+
+def test_parse_links_resolved_via_graph():
+ """With a graph that knows the targets, wikilinks become FileLinks with resolved target_path."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ graph = await _make_graph(
+ FileNode(path="topics/Alice.md", st_mtime=0.0),
+ FileNode(path="topics/Bob.md", st_mtime=0.0),
+ )
+ path = _write_md(tmp, "note.md", "see [[Alice]] and [[Bob#sec]]")
+ parser = LinkedFileParser()
+ parser._resolve_file_graph = lambda: graph
+ node, _ = await parser.parse(path)
+ triples = {(link.target_path, link.target_anchor, link.predicate) for link in node.links}
+ assert ("topics/Alice.md", None, None) in triples
+ assert ("topics/Bob.md", "sec", None) in triples
+ # source_path always equals the node's own path
+ for link in node.links:
+ assert link.source_path == node.path
+ await graph.close()
+ print("✓ test_parse_links_resolved_via_graph passed")
+
+ asyncio.run(run())
+
+
+def test_parse_links_predicate_inline_and_line():
+ """Both `pred:: [[X]]` (line-level) and `[pred:: [[X]]]` (inline) propagate predicate."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ graph = await _make_graph(
+ FileNode(path="A.md", st_mtime=0.0),
+ FileNode(path="B.md", st_mtime=0.0),
+ )
+ body = "extends:: [[A]]\n\nsome [concerns:: [[B]]] inline\n"
+ path = _write_md(tmp, "note.md", body)
+ parser = LinkedFileParser()
+ parser._resolve_file_graph = lambda: graph
+ node, _ = await parser.parse(path)
+ pairs = {(link.target_path, link.predicate) for link in node.links}
+ assert ("A.md", "extends") in pairs
+ assert ("B.md", "concerns") in pairs
+ await graph.close()
+ print("✓ test_parse_links_predicate_inline_and_line passed")
+
+ asyncio.run(run())
+
+
+def test_parse_links_short_path_ambiguity_expands():
+ """A short link matching multiple nodes expands to one FileLink per candidate."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ graph = await _make_graph(
+ FileNode(path="topics/Bob.md", st_mtime=0.0),
+ FileNode(path="people/Bob.md", st_mtime=0.0),
+ )
+ path = _write_md(tmp, "note.md", "ref [[Bob]]")
+ parser = LinkedFileParser()
+ parser._resolve_file_graph = lambda: graph
+ node, _ = await parser.parse(path)
+ targets = sorted(link.target_path for link in node.links)
+ assert targets == ["people/Bob.md", "topics/Bob.md"]
+ await graph.close()
+ print("✓ test_parse_links_short_path_ambiguity_expands passed")
+
+ asyncio.run(run())
+
+
+def test_parse_links_dangling_dropped():
+ """Wikilink to a non-existent target is silently dropped (no graph node)."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ graph = await _make_graph(FileNode(path="topics/Alice.md", st_mtime=0.0))
+ path = _write_md(tmp, "note.md", "[[Alice]] and [[Ghost]]")
+ parser = LinkedFileParser()
+ parser._resolve_file_graph = lambda: graph
+ node, _ = await parser.parse(path)
+ targets = {link.target_path for link in node.links}
+ assert targets == {"topics/Alice.md"}
+ await graph.close()
+ print("✓ test_parse_links_dangling_dropped passed")
+
+ asyncio.run(run())
+
+
+def test_parse_links_deduped():
+ """Repeated wikilinks with the same (target, predicate, anchor) emit one FileLink."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ graph = await _make_graph(FileNode(path="A.md", st_mtime=0.0))
+ path = _write_md(tmp, "note.md", "[[A]] again [[A]] and [[A]]")
+ parser = LinkedFileParser()
+ parser._resolve_file_graph = lambda: graph
+ node, _ = await parser.parse(path)
+ assert len([link for link in node.links if link.target_path == "A.md"]) == 1
+ await graph.close()
+ print("✓ test_parse_links_deduped passed")
+
+ asyncio.run(run())
+
+
+def test_parse_min_chunk_chars_clamped():
+ """chunk_chars below 100 should be clamped to 100."""
+ parser = LinkedFileParser(chunk_chars=10)
+ assert parser.chunk_chars == 100
+ print("✓ test_parse_min_chunk_chars_clamped passed")
+
+
+def test_parse_embed_toc_prefixes_chunk_text():
+ """When embed_toc=True, chunks emitted inside a section are prefixed by the heading."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ body = "# Top\n\n## Sub\n\nbody-content"
+ path = _write_md(tmp, "toc.md", body)
+ parser = LinkedFileParser(chunk_chars=200, embed_toc=True)
+ _, chunks = await parser.parse(path)
+ # Single small section fits; check that the heading appears in text.
+ assert any("Top" in c.text for c in chunks)
+ print("✓ test_parse_embed_toc_prefixes_chunk_text passed")
+
+ asyncio.run(run())
+
+
+if __name__ == "__main__":
+ print("\n=== LinkedFileParser tests ===")
+ test_parse_empty_file()
+ test_parse_frontmatter_only()
+ test_parse_small_body_one_chunk()
+ test_parse_oversized_body_splits()
+ test_parse_chunk_ids_match_node_chunk_ids()
+ test_parse_links_empty_when_no_graph()
+ test_parse_links_resolved_via_graph()
+ test_parse_links_predicate_inline_and_line()
+ test_parse_links_short_path_ambiguity_expands()
+ test_parse_links_dangling_dropped()
+ test_parse_links_deduped()
+ test_parse_min_chunk_chars_clamped()
+ test_parse_embed_toc_prefixes_chunk_text()
+ print("\n所有测试通过!")
diff --git a/tests4/unittest/test_neo4j_file_graph.py b/tests4/unittest/test_neo4j_file_graph.py
new file mode 100644
index 00000000..fef9ef4a
--- /dev/null
+++ b/tests4/unittest/test_neo4j_file_graph.py
@@ -0,0 +1,338 @@
+"""Tests for Neo4jFileGraph.
+
+Skipped automatically if (a) the ``neo4j`` driver isn't installed,
+or (b) a Neo4j instance isn't reachable at the configured URI.
+
+Override the URI / auth via env vars:
+ NEO4J_URI (default ``bolt://localhost:7687``)
+ NEO4J_USER (default ``neo4j``)
+ NEO4J_PASSWORD (default ``neo4j``)
+ NEO4J_DATABASE (default ``neo4j``)
+"""
+
+# pylint: disable=protected-access
+
+import asyncio
+import os
+import tempfile
+
+import pytest
+
+from reme4.schema import FileLink, FileNode
+
+
+URI = os.environ.get("NEO4J_URI", "bolt://localhost:7687")
+USER = os.environ.get("NEO4J_USER", "neo4j")
+PASSWORD = os.environ.get("NEO4J_PASSWORD", "neo4j")
+DATABASE = os.environ.get("NEO4J_DATABASE", "neo4j")
+
+
+try:
+ from reme4.components.file_graph import Neo4jFileGraph
+
+ _NEO4J_IMPORT_ERROR: Exception | None = None
+except Exception as e: # pragma: no cover
+ _NEO4J_IMPORT_ERROR = e
+
+
+async def _probe_neo4j() -> str | None:
+ """Try to connect; return None if OK, error string if unreachable."""
+ if _NEO4J_IMPORT_ERROR is not None:
+ return f"import failed: {_NEO4J_IMPORT_ERROR}"
+ try:
+ from neo4j import AsyncGraphDatabase
+ except ImportError as e:
+ return f"neo4j driver not installed: {e}"
+ try:
+ driver = AsyncGraphDatabase.driver(URI, auth=(USER, PASSWORD))
+ async with driver.session(database=DATABASE) as session:
+ await session.run("RETURN 1")
+ await driver.close()
+ return None
+ except Exception as e: # pragma: no cover
+ return f"connect failed: {e}"
+
+
+_PROBE_NOT_RUN = object()
+_PROBE_REASON: str | None | object = _PROBE_NOT_RUN # sentinel: "not probed"
+
+
+def _probe_reason() -> str | None:
+ """Probe Neo4j once per process; cache the outcome."""
+ global _PROBE_REASON
+ if _PROBE_REASON is _PROBE_NOT_RUN:
+ _PROBE_REASON = asyncio.run(_probe_neo4j())
+ return _PROBE_REASON # type: ignore[return-value]
+
+
+pytestmark = pytest.mark.skipif(
+ _probe_reason() is not None,
+ reason=f"Neo4j unavailable: {_probe_reason()}",
+)
+
+
+class temp_chdir:
+ """Context manager to temporarily chdir into a path and restore on exit."""
+
+ def __init__(self, path):
+ self.path = path
+ self.old = None
+
+ def __enter__(self):
+ self.old = os.getcwd()
+ os.chdir(self.path)
+ return self
+
+ def __exit__(self, *exc):
+ os.chdir(self.old)
+
+
+def make_node(path: str, links: list[tuple[str, str | None]] | None = None) -> FileNode:
+ """Build a FileNode with outgoing (target_path, target_anchor) pairs."""
+ return FileNode(
+ path=path,
+ st_mtime=1.0,
+ links=[FileLink(source_path=path, target_path=t, target_anchor=a) for t, a in (links or [])],
+ )
+
+
+async def _fresh_graph() -> "Neo4jFileGraph": # type: ignore[name-defined]
+ """Build a started Neo4jFileGraph wiped clean."""
+ graph = Neo4jFileGraph(uri=URI, user=USER, password=PASSWORD, database=DATABASE)
+ await graph.start()
+ await graph.clear()
+ return graph
+
+
+def test_upsert_and_get_nodes():
+ """upsert_nodes stores; get_nodes returns by paths or all."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ graph = await _fresh_graph()
+ try:
+ await graph.upsert_nodes(
+ [make_node("a.md", [("b.md", None)]), make_node("b.md")],
+ )
+ got_all = await graph.get_nodes()
+ assert {n.path for n in got_all} == {"a.md", "b.md"}
+ got_one = await graph.get_nodes(["a.md"])
+ assert len(got_one) == 1 and got_one[0].path == "a.md"
+ assert await graph.get_nodes(["nope.md"]) == []
+ assert await graph.get_nodes([]) == []
+ finally:
+ await graph.clear()
+ await graph.close()
+ print("✓ test_upsert_and_get_nodes passed")
+
+ asyncio.run(run())
+
+
+def test_outlinks_skip_virtual_targets():
+ """get_outlinks excludes edges into virtual placeholder nodes."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ graph = await _fresh_graph()
+ try:
+ await graph.upsert_nodes(
+ [
+ make_node("a.md", [("b.md", None), ("ghost.md", None)]),
+ make_node("b.md"),
+ ],
+ )
+ outs = await graph.get_outlinks("a.md")
+ assert {link.target_path for link in outs} == {"b.md"}
+ for link in outs:
+ assert link.source_path == "a.md"
+ finally:
+ await graph.clear()
+ await graph.close()
+ print("✓ test_outlinks_skip_virtual_targets passed")
+
+ asyncio.run(run())
+
+
+def test_inlinks_carry_source_path():
+ """get_inlinks returns FileLinks whose source_path is the linking node."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ graph = await _fresh_graph()
+ try:
+ await graph.upsert_nodes(
+ [
+ make_node("a.md", [("b.md", "anchor1")]),
+ make_node("c.md", [("b.md", None)]),
+ make_node("b.md"),
+ ],
+ )
+ ins = await graph.get_inlinks("b.md")
+ sources = {link.source_path for link in ins}
+ assert sources == {"a.md", "c.md"}
+ # Each link's target should be the queried path.
+ for link in ins:
+ assert link.target_path == "b.md"
+ finally:
+ await graph.clear()
+ await graph.close()
+ print("✓ test_inlinks_carry_source_path passed")
+
+ asyncio.run(run())
+
+
+def test_delete_demotes_then_repromotes():
+ """delete_nodes makes a node virtual; re-upsert promotes pending edges back."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ graph = await _fresh_graph()
+ try:
+ await graph.upsert_nodes(
+ [make_node("a.md", [("b.md", None)]), make_node("b.md")],
+ )
+ assert {link.source_path for link in await graph.get_inlinks("b.md")} == {"a.md"}
+
+ await graph.delete_nodes(["b.md"])
+ assert await graph.get_nodes(["b.md"]) == []
+ # a's outlink is hidden because b is now virtual.
+ assert await graph.get_outlinks("a.md") == []
+
+ await graph.upsert_nodes([make_node("b.md")])
+ assert {link.source_path for link in await graph.get_inlinks("b.md")} == {"a.md"}
+ finally:
+ await graph.clear()
+ await graph.close()
+ print("✓ test_delete_demotes_then_repromotes passed")
+
+ asyncio.run(run())
+
+
+def test_rebuild_links_idempotent():
+ """rebuild_links reconstructs identical out/in views from per-node payloads."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ graph = await _fresh_graph()
+ try:
+ await graph.upsert_nodes(
+ [
+ make_node("a.md", [("b.md", None), ("c.md", "h")]),
+ make_node("b.md"),
+ make_node("c.md"),
+ ],
+ )
+ before_out = sorted((link.target_path, link.target_anchor) for link in await graph.get_outlinks("a.md"))
+ before_in_b = sorted(link.source_path for link in await graph.get_inlinks("b.md"))
+
+ await graph.rebuild_links()
+
+ after_out = sorted((link.target_path, link.target_anchor) for link in await graph.get_outlinks("a.md"))
+ after_in_b = sorted(link.source_path for link in await graph.get_inlinks("b.md"))
+ assert before_out == after_out
+ assert before_in_b == after_in_b
+ finally:
+ await graph.clear()
+ await graph.close()
+ print("✓ test_rebuild_links_idempotent passed")
+
+ asyncio.run(run())
+
+
+def test_clear_wipes_everything():
+ """clear() drops every node and edge in the database."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ graph = await _fresh_graph()
+ try:
+ await graph.upsert_nodes(
+ [make_node("a.md", [("b.md", None)]), make_node("b.md")],
+ )
+ await graph.clear()
+ assert await graph.get_nodes() == []
+ finally:
+ await graph.close()
+ print("✓ test_clear_wipes_everything passed")
+
+ asyncio.run(run())
+
+
+def test_node_roundtrip_preserves_frontmatter_and_links():
+ """Upsert → get_nodes round-trip preserves frontmatter + links + chunk_ids."""
+
+ async def run():
+ from reme4.schema.file_node import FileFrontMatter
+
+ with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+ graph = await _fresh_graph()
+ try:
+ node = FileNode(
+ path="topics/Alice.md",
+ st_mtime=1234.5,
+ links=[
+ FileLink(
+ source_path="topics/Alice.md",
+ target_path="topics/Bob.md",
+ target_anchor="intro",
+ predicate="knows",
+ ),
+ ],
+ chunk_ids=["chunk-a1", "chunk-a2", "chunk-a3"],
+ front_matter=FileFrontMatter(
+ title="Alice",
+ description="a person",
+ tags=["x", "y"],
+ ),
+ )
+ await graph.upsert_nodes([node, make_node("topics/Bob.md")])
+ got = await graph.get_nodes(["topics/Alice.md"])
+ assert len(got) == 1
+ back = got[0]
+ assert back.path == "topics/Alice.md"
+ assert back.st_mtime == 1234.5
+ assert back.front_matter.title == "Alice"
+ assert back.front_matter.description == "a person"
+ assert sorted(back.front_matter.tags or []) == ["x", "y"]
+ assert back.chunk_ids == ["chunk-a1", "chunk-a2", "chunk-a3"]
+ assert len(back.links) == 1
+ link = back.links[0]
+ assert link.source_path == "topics/Alice.md"
+ assert link.target_path == "topics/Bob.md"
+ assert link.target_anchor == "intro"
+ assert link.predicate == "knows"
+
+ # An upsert with empty chunk_ids should also round-trip cleanly
+ # (and overwrite the previous list).
+ await graph.upsert_nodes(
+ [
+ FileNode(
+ path="topics/Alice.md",
+ st_mtime=1234.5,
+ chunk_ids=[],
+ ),
+ ],
+ )
+ got2 = await graph.get_nodes(["topics/Alice.md"])
+ assert got2 and got2[0].chunk_ids == []
+ finally:
+ await graph.clear()
+ await graph.close()
+ print("✓ test_node_roundtrip_preserves_frontmatter_and_links passed")
+
+ asyncio.run(run())
+
+
+if __name__ == "__main__":
+ if _probe_reason() is not None:
+ print(f"Skipping Neo4j tests: {_probe_reason()}")
+ else:
+ print("\n=== Neo4jFileGraph tests ===")
+ test_upsert_and_get_nodes()
+ test_outlinks_skip_virtual_targets()
+ test_inlinks_carry_source_path()
+ test_delete_demotes_then_repromotes()
+ test_rebuild_links_idempotent()
+ test_clear_wipes_everything()
+ test_node_roundtrip_preserves_frontmatter_and_links()
+ print("\n所有测试通过!")
From 68bd95b494e46518d600e954e39701d6e398f423 Mon Sep 17 00:00:00 2001
From: jinliyl <6469360+jinliyl@users.noreply.github.com>
Date: Mon, 18 May 2026 15:00:02 +0800
Subject: [PATCH 12/16] =?UTF-8?q?refactor(steps):=20Add=20job=20management?=
=?UTF-8?q?=20methods=20and=20support=20registering=20t=E2=80=A6=20(#242)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* refactor(steps): Add job management methods and support registering them as tools
Added methods to the `BaseStep` class for retrieving, running, and registering jobs as tools, enhancing the functionality of the step class.
* fix doc
---
reme4/steps/base_step.py | 34 ++++++++++++++++++++++++++++++++++
1 file changed, 34 insertions(+)
diff --git a/reme4/steps/base_step.py b/reme4/steps/base_step.py
index a710f457..d32c4445 100644
--- a/reme4/steps/base_step.py
+++ b/reme4/steps/base_step.py
@@ -5,8 +5,10 @@ from abc import abstractmethod, ABC
from typing import TypeVar, TYPE_CHECKING
from agentscope.formatter import FormatterBase
+from agentscope.message import TextBlock
from agentscope.model import ChatModelBase
from agentscope.token import TokenCounterBase
+from agentscope.tool import Toolkit, ToolResponse
from ..components.embedding import BaseEmbeddingModel
from ..components.file_parser import BaseFileParser
@@ -15,10 +17,12 @@ from ..components.file_watcher import BaseFileWatcher
from ..components.prompt_handler import PromptHandler
from ..components.runtime_context import RuntimeContext
from ..enumeration import ComponentEnum
+from ..schema import Response
from ..utils import get_logger
if TYPE_CHECKING:
from ..components import ApplicationContext
+ from ..components.job import BaseJob
T = TypeVar("T")
@@ -138,3 +142,33 @@ class BaseStep(ABC):
def copy(self, **kwargs) -> "BaseStep":
"""Construct a new instance from the original init args, applying overrides."""
return self.__class__(*self._init_args, **{**self._init_kwargs, **kwargs})
+
+ def get_job(self, name: str) -> "BaseJob | None":
+ """Return a job by name."""
+ if self.app_context is None:
+ raise RuntimeError("Cannot get job without an app context")
+ return self.app_context.jobs.get(name)
+
+ async def run_job(self, name: str, **kwargs) -> Response:
+ """Execute a job by name and kwargs, return the final response."""
+ job: "BaseJob | None" = self.get_job(name)
+ if job is None:
+ raise RuntimeError(f"Job {name} not found")
+ return await job(**kwargs)
+
+ def add_as_tool(self, toolkit: Toolkit, job_name: str) -> None:
+ """Add the step as a tool to the toolkit."""
+ job: "BaseJob | None" = self.get_job(job_name)
+ if job is None:
+ raise RuntimeError(f"Job {job_name} not found")
+
+ async def run_job(**kwargs) -> ToolResponse:
+ response = await job(**kwargs)
+ return ToolResponse(content=[TextBlock(type="text", text=response.answer)])
+
+ toolkit.register_tool_function(
+ tool_func=run_job,
+ func_name=job_name,
+ func_description=job.description,
+ json_schema=job.parameters,
+ )
From 0e5a9f50346f6d5a19251e278a7bb20d3d2355f9 Mon Sep 17 00:00:00 2001
From: Joshua <75612919+Joshuaakaspace@users.noreply.github.com>
Date: Mon, 18 May 2026 12:48:23 +0530
Subject: [PATCH 13/16] fix(reme_light): dedupe default watch paths on
case-insensitive filesystems (#234)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* fix(reme_light): dedupe default watch paths on case-insensitive filesystems
On Windows NTFS and macOS HFS+, ``MEMORY.md`` and ``memory.md`` resolve to
the same physical file. ``ReMeLight.__init__`` hardcoded both spellings in
the default ``watch_paths`` list, so the memory markdown file was indexed
twice on those filesystems, wasting embedding calls and producing duplicate
search hits.
Dedupe the default candidate list using ``os.path.normcase`` as the
comparison key. On case-sensitive filesystems normcase is the identity
function, so both spellings continue to be watched there. The original
path strings are preserved, the caller-supplied ``watch_paths`` path is
untouched, and only the built-in fallback is affected.
Fixes #228
* refactor(reme_light): simplify watch path dedup via existence check
Replace the os.path.normcase-based dedup loop with a direct exists()
check that picks one of MEMORY.md / memory.md. On case-insensitive
filesystems both spellings resolve to the same file so exists() returns
true for both, naturally avoiding a duplicate watch — including on
macOS where os.path.normcase is the identity function and the previous
approach silently did nothing.
Co-Authored-By: Claude Opus 4.7
---------
Co-authored-by: jinli.yl
Co-authored-by: Claude Opus 4.7
---
reme/reme_light.py | 14 +++---
tests/test_reme_light_watch_paths.py | 67 ++++++++++++++++++++++++++++
2 files changed, 75 insertions(+), 6 deletions(-)
create mode 100644 tests/test_reme_light_watch_paths.py
diff --git a/reme/reme_light.py b/reme/reme_light.py
index 91e738ed..ca52a394 100644
--- a/reme/reme_light.py
+++ b/reme/reme_light.py
@@ -135,12 +135,14 @@ class ReMeLight(Application):
self.vector_weight: float = vector_weight
self.candidate_multiplier: float = candidate_multiplier
- # Build the file watcher config: use provided watch_paths if given, otherwise use defaults
- _default_watch_paths = [
- str(self.working_path / "MEMORY.md"),
- str(self.working_path / "memory.md"),
- str(self.memory_path),
- ]
+ # Pick the existing memory markdown file. On case-insensitive filesystems
+ # (Windows NTFS, macOS APFS/HFS+) ``MEMORY.md`` and ``memory.md`` are the
+ # same file, so this also avoids watching it twice. Default to ``MEMORY.md``
+ # when neither exists yet.
+ _memory_md = self.working_path / "MEMORY.md"
+ if not _memory_md.exists() and (self.working_path / "memory.md").exists():
+ _memory_md = self.working_path / "memory.md"
+ _default_watch_paths = [str(_memory_md), str(self.memory_path)]
if default_file_watcher_config and default_file_watcher_config.get("watch_paths"):
_merged_file_watcher_config = default_file_watcher_config
else:
diff --git a/tests/test_reme_light_watch_paths.py b/tests/test_reme_light_watch_paths.py
new file mode 100644
index 00000000..4d0a91f9
--- /dev/null
+++ b/tests/test_reme_light_watch_paths.py
@@ -0,0 +1,67 @@
+"""
+Tests for the default watch path construction in ``ReMeLight``.
+
+Verifies that the built-in watch list picks a single ``MEMORY.md`` /
+``memory.md`` spelling so the file is not indexed twice on case-insensitive
+filesystems (Windows NTFS, macOS APFS/HFS+). See agentscope-ai/ReMe#228.
+"""
+
+# pylint: disable=redefined-outer-name,protected-access,missing-function-docstring,missing-class-docstring
+
+import tempfile
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from reme.reme_light import ReMeLight
+
+
+@pytest.fixture
+def temp_working_dir():
+ with tempfile.TemporaryDirectory() as tmp:
+ yield tmp
+
+
+def _captured_watch_paths(working_dir: str, *, default_file_watcher_config=None):
+ """Capture the ``watch_paths`` that ``ReMeLight`` would forward to its
+ parent ``Application.__init__``, without spinning up the full app stack."""
+ captured: dict = {}
+
+ def _capture(*_args, **kwargs):
+ captured.update(kwargs)
+
+ with patch("reme.reme_light.Application.__init__", _capture):
+ ReMeLight(
+ working_dir=working_dir,
+ default_file_watcher_config=default_file_watcher_config,
+ )
+
+ return list((captured.get("default_file_watcher_config") or {}).get("watch_paths", []))
+
+
+class TestDefaultWatchPaths:
+ def test_defaults_to_uppercase_memory_md_when_neither_exists(self, temp_working_dir):
+ paths = _captured_watch_paths(temp_working_dir)
+ memory_dir = str(Path(temp_working_dir).absolute() / "memory")
+ assert paths == [str(Path(temp_working_dir).absolute() / "MEMORY.md"), memory_dir]
+
+ def test_picks_lowercase_memory_md_when_only_it_exists(self, temp_working_dir):
+ (Path(temp_working_dir) / "memory.md").write_text("")
+ if (Path(temp_working_dir) / "MEMORY.md").exists():
+ pytest.skip("case-insensitive filesystem treats both spellings as one file")
+ paths = _captured_watch_paths(temp_working_dir)
+ assert paths[0] == str(Path(temp_working_dir).absolute() / "memory.md")
+
+ def test_prefers_uppercase_memory_md_when_it_exists(self, temp_working_dir):
+ (Path(temp_working_dir) / "MEMORY.md").write_text("")
+ paths = _captured_watch_paths(temp_working_dir)
+ assert paths[0] == str(Path(temp_working_dir).absolute() / "MEMORY.md")
+
+ def test_user_provided_watch_paths_pass_through(self, temp_working_dir):
+ custom = [str(Path(temp_working_dir) / "notes.md")]
+ paths = _captured_watch_paths(
+ temp_working_dir,
+ default_file_watcher_config={"watch_paths": custom},
+ )
+ assert paths == custom
From b001c060869e4f5c7a36d7ed65aef320c689a4a8 Mon Sep 17 00:00:00 2001
From: jinliyl <6469360+jinliyl@users.noreply.github.com>
Date: Mon, 18 May 2026 15:53:27 +0800
Subject: [PATCH 14/16] Dev/connect as tool (#243)
* refactor(steps): Add job management methods and support registering them as tools
Added methods to the `BaseStep` class for retrieving, running, and registering jobs as tools, enhancing the functionality of the step class.
* fix doc
* chore(pyproject.toml): Update dependency versions and adjust package configuration
Bump agentscope version to 1.0.19 and reorganize the core dependency configuration structure.
---
pyproject.toml | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 81039167..89d9020e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -93,11 +93,15 @@ litellm = [
"litellm==1.80.0",
]
-core = [
- "agentscope==1.0.18",
+light = [
+ "agentscope==1.0.19",
"flowllm[reme]>=0.2.0.10",
]
+core = [
+ "agentscope==1.0.19",
+]
+
[tool.setuptools.packages.find]
where = ["."]
include = ["reme_ai*", "reme*", "reme4*"]
From bee2648ad12234d9bac924ddfdcdaa49c1c8510b Mon Sep 17 00:00:00 2001
From: imrewce
Date: Tue, 19 May 2026 16:19:13 +0800
Subject: [PATCH 15/16] feat: implementation of the read step for reme
(markdown) (#245)
* feat: implementation of the read step for reme (markdown)
* fix(step): markdown read step fixing pr comments
* fix(step): more fixes for pr comments
* further fix for better review adaptation
* accept absolute path
* fixing base job exception
---
docs4/reme_design.md | 6 +
reme4/components/job/base_job.py | 1 +
reme4/config/default.yaml | 20 ++
reme4/constants.py | 5 +
reme4/steps/__init__.py | 2 +
reme4/steps/base_step.py | 16 +-
reme4/steps/crud/__init__.py | 7 +
reme4/steps/crud/_file_io.py | 109 ++++++++
reme4/steps/crud/read.py | 91 +++++++
tests4/unittest/test_crud_md_steps.py | 371 ++++++++++++++++++++++++++
10 files changed, 627 insertions(+), 1 deletion(-)
create mode 100644 reme4/steps/crud/__init__.py
create mode 100644 reme4/steps/crud/_file_io.py
create mode 100644 reme4/steps/crud/read.py
create mode 100644 tests4/unittest/test_crud_md_steps.py
diff --git a/docs4/reme_design.md b/docs4/reme_design.md
index 15e7ee28..fab618ff 100644
--- a/docs4/reme_design.md
+++ b/docs4/reme_design.md
@@ -42,6 +42,7 @@ reme4 version
| 🔎 search | 🔍 `search` (`search_step`) | `call_server("search", query=…, …)` | 📥 `query:str` ⭐ ｜ 🎚️ `limit:int=5`(>0) ｜ 🎚️ `min_score:float=0.0` ｜ ⚖️ `vector_weight:float=0.7` ∈[0,1]（keyword 权 = 1-vw）｜ 🔀 `candidate_multiplier:float=3.0`（candidates = min(200, limit×mult)）｜ 🔗 `expand_links:bool=True` ｜ 🔢 `max_links_per_direction:int=10` ｜ 🎚️ `search_filter:dict={}` ｜ 📤 `answer` 每命中一行 `path:start-end [score=… vector=… keyword=…] text` + 缩进的 `→ outlinks (n)` / `← inlinks (n)` + `via predicate=… anchor=#…` ｜ 📊 `metadata.results` / `metadata.link_expansion` / `metadata.counts={vector,keyword,returned,hybrid}` ｜ 🛠️ 并行 `vector_search` + `keyword_search` → RRF 融合（K=60，按 chunk.id 合并）→ `min_score` 过滤 → `limit` 截断 → 邻居 meta 注入 |
| 🧪 demo | 🪄 `demo_echo` (`demo_echo_step1` + `step2`) | `call_server("demo_echo", query=…, min_score=…)` | 📥 `query:str=""` ｜ 🎚️ `min_score:float=0.5` ｜ 🛠️ step1：`processed_query = query.strip().lower()`，`adjusted_min_score = min_score * 0.9`，写回 context ｜ 📤 step2：`answer = "echo: {processed_query} (min_score={adjusted_min_score})"` ｜ 📊 `metadata = {step, query, min_score, processed_query, adjusted_min_score}` |
| 🌊 demo | 🌊 `stream_demo` (`stream_demo_step1` + `step2`) | `call_server("stream_demo", query=…, repeat=…, interval=…)` | 📥 `query:str=""` ｜ 🎚️ `repeat:int=10` ｜ 🎚️ `interval:float=0.1`（秒/字符）｜ 🛠️ step1：`stream_text = query * repeat` 写回 context ｜ 📤 step2：按字符 `add_stream_string(ch, ChunkEnum.CONTENT)` 流式输出，`asyncio.sleep(interval)` 节流 |
+| 📂 crud | 📖 `read` (`read_step`) | `call_server("read", path=…, …)` | 📥 `path:str` ⭐（**完整相对路径**，相对于 `working_dir`；绝对路径会被拒绝；非 `.md` 后缀拒绝）｜ 🎚️ `start_line:int=null`（1-based, 含端点）｜ 🎚️ `end_line:int=null`（1-based, 含端点）｜ 🎚️ `max_bytes:int=51200`（截断阈值）｜ 📤 `answer = 选中的行内容`，超过 `max_bytes` 时附加 `--- TRUNCATED ---` 续读指引（`start_line=…`）｜ 📊 `metadata.path` / `metadata.total_lines`（出错路径才会附带）｜ 🛠️ 流程：`BaseStep.resolve_path(raw, require_md=True)` → `aiofiles.os.stat` → `read_file_safe`（utf-8-sig BOM 容忍、UnicodeDecodeError fallback `errors=ignore`）→ `split("\n")` 切片 `[s-1:e]` → `truncate_text_output` 按字节截断保行 |
使用示例：
@@ -66,6 +67,11 @@ reme4 version
reme4 reindex
reme4 search query="latency 问题" limit=10 min_score=0.2 vector_weight=0.6
+# 读取 working_dir 下的 markdown（完整相对路径；无后缀自动补 .md；可按行切片或限制字节）
+reme4 read path=Templates/Recipe.md
+reme4 read path=Notes start_line=1 end_line=20
+reme4 read path=Big.md max_bytes=4096
+
# 通过 MCP backend 调用
reme4 search query="..." backend=mcp
```
diff --git a/reme4/components/job/base_job.py b/reme4/components/job/base_job.py
index e4681bf1..2583989f 100644
--- a/reme4/components/job/base_job.py
+++ b/reme4/components/job/base_job.py
@@ -49,5 +49,6 @@ class BaseJob(BaseComponent):
await step(context)
except Exception as e:
self.logger.exception(f"Failed to execute job: {e}")
+ context.response.success = False
context.response.answer = str(e)
return context.response
diff --git a/reme4/config/default.yaml b/reme4/config/default.yaml
index 1a501912..6e543ec9 100644
--- a/reme4/config/default.yaml
+++ b/reme4/config/default.yaml
@@ -96,6 +96,26 @@ jobs:
steps:
- backend: search_step
+ - backend: base
+ name: read
+ description: "read a markdown file (relative path under working_dir)"
+ parameters:
+ type: object
+ properties:
+ path:
+ type: string
+ description: "relative path under the working_dir (no absolute paths); markdown only"
+ start_line:
+ type: integer
+ description: "Optional, first line to read (1-based, inclusive)"
+ end_line:
+ type: integer
+ description: "Optional, last line to read (1-based, inclusive)"
+ required:
+ - path
+ steps:
+ - backend: read_step
+
- backend: stream
name: stream_demo
description: "stream demo job: repeat query 10x and stream char-by-char"
diff --git a/reme4/constants.py b/reme4/constants.py
index fce66e64..819fbadd 100644
--- a/reme4/constants.py
+++ b/reme4/constants.py
@@ -5,3 +5,8 @@ REME_SERVICE_INFO = "REME_SERVICE_INFO"
REME_DEFAULT_HOST = "127.0.0.1"
REME_DEFAULT_PORT = 2333
+
+# CRUD steps: file IO limits and truncation marker (shared across CRUD steps).
+DEFAULT_MAX_BYTES = 50 * 1024
+MAX_FILE_READ_BYTES = 200 * 1024 * 1024
+TRUNCATION_NOTICE_MARKER = "<>"
diff --git a/reme4/steps/__init__.py b/reme4/steps/__init__.py
index fa25273c..70878e5f 100644
--- a/reme4/steps/__init__.py
+++ b/reme4/steps/__init__.py
@@ -1,9 +1,11 @@
"""steps"""
from . import common
+from . import crud
from .base_step import BaseStep
__all__ = [
"common",
+ "crud",
"BaseStep",
]
diff --git a/reme4/steps/base_step.py b/reme4/steps/base_step.py
index d32c4445..f8a4d7aa 100644
--- a/reme4/steps/base_step.py
+++ b/reme4/steps/base_step.py
@@ -2,6 +2,7 @@
import copy
from abc import abstractmethod, ABC
+from pathlib import Path
from typing import TypeVar, TYPE_CHECKING
from agentscope.formatter import FormatterBase
@@ -83,7 +84,20 @@ class BaseStep(ABC):
self.context.apply_mapping(self.output_mapping)
return result
- def _resolve(self, key: str, base_cls: type[T], comp_enum: ComponentEnum, attr: str | None = None) -> T:
+ @property
+ def working_path(self) -> Path:
+ """Resolved working directory from app context or cwd."""
+ if self.app_context is None:
+ return Path.cwd()
+ return Path(self.app_context.app_config.working_dir)
+
+ def _resolve(
+ self,
+ key: str,
+ base_cls: type[T],
+ comp_enum: ComponentEnum,
+ attr: str | None = None,
+ ) -> T:
"""Return a kwargs-supplied instance, or look one up by name in the app registry."""
# 1. Step init kwargs, 2. Runtime context (run_job kwargs), 3. App registry by name.
for source in (self.kwargs, self.context or {}):
diff --git a/reme4/steps/crud/__init__.py b/reme4/steps/crud/__init__.py
new file mode 100644
index 00000000..de9ae9ce
--- /dev/null
+++ b/reme4/steps/crud/__init__.py
@@ -0,0 +1,7 @@
+"""CRUD steps for markdown files under the working_dir."""
+
+from .read import ReadStep
+
+__all__ = [
+ "ReadStep",
+]
diff --git a/reme4/steps/crud/_file_io.py b/reme4/steps/crud/_file_io.py
new file mode 100644
index 00000000..4beaa35d
--- /dev/null
+++ b/reme4/steps/crud/_file_io.py
@@ -0,0 +1,109 @@
+"""Shared filesystem helpers for CRUD steps (path gating, safe read, truncation)."""
+
+from pathlib import Path
+
+import aiofiles
+import aiofiles.os
+
+from ...constants import DEFAULT_MAX_BYTES, MAX_FILE_READ_BYTES, TRUNCATION_NOTICE_MARKER
+from ...utils import get_logger
+
+logger = get_logger()
+
+
+def resolve_path(working_path: Path, raw: str) -> tuple[Path | None, str | None]:
+ """Resolve a relative `path=` argument under self.working_path.
+
+ Rules:
+ - the caller supplies the full relative path under ``self.working_path``;
+ absolute paths are rejected.
+ Returns ``(abs_path, None)`` on success, or ``(None, error_message)`` on failure.
+ Filetype-specific gating (e.g. markdown-only / suffix auto-append) is
+ layered on top by callers — see ``reme4/steps/crud/_file_io.py::gate_md``.
+ """
+ if not raw or not str(raw).strip():
+ return None, "`path` is required"
+ s = str(raw).strip()
+ p = Path(s)
+ if p.is_absolute():
+ logger.info("absolute path detected, recommmending relative paths")
+ return p, None
+ return working_path / p, None
+
+
+def gate_md(target: Path, raw: str) -> tuple[Path | None, str | None]:
+ """Markdown-only gate: auto-append `.md` when no suffix; reject any non-`.md` suffix.
+
+ Layered on top of ``BaseStep.resolve_path`` to keep filetype-specific rules
+ out of the generic path resolver.
+ """
+ if target.suffix == "":
+ return target.with_suffix(".md"), None
+ if target.suffix.lower() != ".md":
+ return None, (f"path {raw!r} is not a markdown file; this command only supports .md files")
+ return target, None
+
+
+async def read_file_safe(file_path, max_bytes: int = MAX_FILE_READ_BYTES) -> str:
+ """Read file with utf-8-sig (BOM-tolerant), fallback to errors='ignore'."""
+ stat = await aiofiles.os.stat(str(file_path))
+ read_size = min(stat.st_size, max_bytes)
+ try:
+ async with aiofiles.open(str(file_path), "r", encoding="utf-8-sig") as f:
+ return await f.read(read_size)
+ except UnicodeDecodeError:
+ async with aiofiles.open(
+ str(file_path),
+ "r",
+ encoding="utf-8-sig",
+ errors="ignore",
+ ) as f:
+ return await f.read(read_size)
+
+
+def truncate_text_output(
+ text: str,
+ *,
+ start_line: int = 1,
+ total_lines: int = 0,
+ max_bytes: int = DEFAULT_MAX_BYTES,
+ file_path: str | None = None,
+ encoding: str = "utf-8",
+) -> str:
+ """Truncate text by bytes preserving line integrity; append a continuation notice.
+
+ See qwenpaw `tools/utils.py` for the same semantics. Returns text unchanged when
+ it fits within max_bytes, when max_bytes <= 0, or when the last line itself
+ exceeds max_bytes (unhandled edge case).
+ """
+ if not text or max_bytes <= 0:
+ return text
+
+ try:
+ text_bytes = text.encode(encoding)
+ if len(text_bytes) <= max_bytes:
+ return text
+
+ truncated = text_bytes[:max_bytes]
+ result = truncated.decode(encoding, errors="ignore")
+ newline_count = result.count("\n")
+ next_line = start_line + max(1, newline_count)
+
+ if next_line <= total_lines:
+ read_from = next_line
+ elif start_line < total_lines:
+ read_from = total_lines
+ else:
+ return result
+
+ notice = (
+ TRUNCATION_NOTICE_MARKER + f"\nThe output above was truncated."
+ f"\nThe full content is saved to the file and contains {total_lines} lines in total."
+ f"\nThis excerpt starts at line {start_line} and covers the next {max_bytes} bytes."
+ f"\nIf the current content is not enough, call `read` with file={file_path or ''} "
+ f"start_line={read_from} to read more."
+ )
+ return result + notice
+ except Exception:
+ logger.warning("truncate_text_output failed, returning original text", exc_info=True)
+ return text
diff --git a/reme4/steps/crud/read.py b/reme4/steps/crud/read.py
new file mode 100644
index 00000000..55285efa
--- /dev/null
+++ b/reme4/steps/crud/read.py
@@ -0,0 +1,91 @@
+"""Read a markdown file from the vault, with line-range slicing and byte-truncation."""
+
+from ._file_io import (
+ gate_md,
+ resolve_path,
+ read_file_safe,
+ truncate_text_output,
+)
+from ..base_step import BaseStep
+from ...components import R
+
+
+@R.register("read_step")
+class ReadStep(BaseStep):
+ """Read a markdown file. Optional `start_line`/`end_line` for ranged reads."""
+
+ def _fail(self, message: str, **meta) -> None:
+ assert self.context is not None
+ self.context.response.success = False
+ self.context.response.answer = f"Error: {message}"
+ if meta:
+ self.context.response.metadata.update(meta)
+
+ async def execute(self): # pylint: disable=too-many-return-statements
+ assert self.context is not None
+ raw = str(self.context.get("path") or "")
+ start_line = self.context.get("start_line")
+ end_line = self.context.get("end_line")
+
+ target, err = resolve_path(self.working_path, raw)
+ if err:
+ self._fail(err)
+ return None
+
+ target, err = gate_md(target, raw)
+ if err:
+ self._fail(err)
+ return None
+
+ for label, value in (("start_line", start_line), ("end_line", end_line)):
+ if value is None:
+ continue
+ try:
+ int(value)
+ except (TypeError, ValueError):
+ self._fail(f"{label} must be an integer, got {value!r}")
+ return None
+
+ if not target.exists():
+ self._fail(f"file {target} does not exist", path=str(target))
+ return None
+ if not target.is_file():
+ self._fail(f"path {target} is not a file", path=str(target))
+ return None
+
+ try:
+ content = await read_file_safe(target)
+ except Exception as e:
+ self._fail(f"read failed: {e}", path=str(target))
+ return None
+
+ all_lines = content.split("\n")
+ total = len(all_lines)
+ s = max(1, int(start_line) if start_line is not None else 1)
+ e = min(total, int(end_line) if end_line is not None else total)
+
+ if s > total:
+ self._fail(
+ f"start_line {s} exceeds file length ({total} lines)",
+ path=str(target),
+ total_lines=total,
+ )
+ return None
+ if s > e:
+ self._fail(f"start_line ({s}) > end_line ({e})", path=str(target))
+ return None
+
+ selected = "\n".join(all_lines[s - 1 : e])
+ text = truncate_text_output(
+ selected,
+ start_line=s,
+ total_lines=total,
+ file_path=str(target),
+ )
+
+ self.context.response.success = True
+ self.context.response.answer = text
+ self.logger.info(
+ f"[{self.name}] read path={target} lines={s}-{e}/{total} bytes={len(text.encode('utf-8'))}",
+ )
+ return self.context.response
diff --git a/tests4/unittest/test_crud_md_steps.py b/tests4/unittest/test_crud_md_steps.py
new file mode 100644
index 00000000..d60c72f9
--- /dev/null
+++ b/tests4/unittest/test_crud_md_steps.py
@@ -0,0 +1,371 @@
+"""End-to-end tests for reme4 crud_md steps: spawn `reme4 start`, drive via HTTP,
+verify responses, then shut down. Each test uses an isolated cwd so the working_dir
+(.reme by default) does not collide.
+
+CLI rule: `path=` is relative-only, rooted at the reme working_dir. A bare path with
+no suffix auto-appends `.md`; non-`.md` suffix is rejected. Absolute paths are
+rejected.
+"""
+
+import asyncio
+import os
+import tempfile
+import warnings
+from pathlib import Path
+
+from reme4.utils import call_action, call_and_check, mock_reme_server
+
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba")
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources")
+
+
+class _temp_chdir:
+ """chdir to path for the duration of the block; restore on exit."""
+
+ def __init__(self, path):
+ self.path = path
+ self._old = None
+
+ def __enter__(self):
+ self._old = os.getcwd()
+ os.chdir(self.path)
+ return self
+
+ def __exit__(self, *exc):
+ os.chdir(self._old)
+
+
+def _run(coro):
+ """Run an async coroutine on a fresh isolated event loop."""
+ asyncio.run(coro)
+
+
+def _seed_md(working_dir: Path, rel: str, body: str) -> Path:
+ target = working_dir / rel
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_text(body, encoding="utf-8")
+ return target
+
+
+# ---------------------------------------------------------------------------
+# Individual job tests
+# ---------------------------------------------------------------------------
+
+
+def test_read_relative_path():
+ """`reme4 read path=Templates/Recipe.md` returns the file body from .reme/."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ working = Path(tmp) / ".reme"
+ working.mkdir(parents=True, exist_ok=True)
+ body = "# Recipe\n\nMix flour and water.\n"
+ _seed_md(working, "Templates/Recipe.md", body)
+ async with mock_reme_server() as (host, port):
+ await call_and_check(
+ "read",
+ host=host,
+ port=port,
+ path="Templates/Recipe.md",
+ validator=lambda r: (
+ isinstance(r, dict)
+ and r.get("success") is True
+ and "# Recipe" in str(r.get("answer", ""))
+ and "flour and water" in str(r.get("answer", ""))
+ ),
+ )
+ print("✓ test_read_relative_path passed")
+
+ _run(run())
+
+
+def test_read_no_suffix_autoappends_md():
+ """A bare path with no suffix auto-appends `.md`."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ working = Path(tmp) / ".reme"
+ working.mkdir(parents=True, exist_ok=True)
+ _seed_md(working, "Templates/Recipe.md", "auto-md\n")
+ async with mock_reme_server() as (host, port):
+ await call_and_check(
+ "read",
+ host=host,
+ port=port,
+ path="Templates/Recipe",
+ validator=lambda r: (
+ isinstance(r, dict) and r.get("success") is True and "auto-md" in str(r.get("answer", ""))
+ ),
+ )
+ print("✓ test_read_no_suffix_autoappends_md passed")
+
+ _run(run())
+
+
+def test_read_line_range():
+ """start_line / end_line slice the file 1-based, inclusive."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ working = Path(tmp) / ".reme"
+ working.mkdir(parents=True, exist_ok=True)
+ _seed_md(working, "Notes.md", "L1\nL2\nL3\nL4\nL5\n")
+ async with mock_reme_server() as (host, port):
+ await call_and_check(
+ "read",
+ host=host,
+ port=port,
+ path="Notes.md",
+ start_line=2,
+ end_line=4,
+ validator=lambda r: (
+ isinstance(r, dict)
+ and r.get("success") is True
+ and "L2" in str(r["answer"])
+ and "L3" in str(r["answer"])
+ and "L4" in str(r["answer"])
+ and "L1" not in str(r["answer"])
+ and "L5" not in str(r["answer"])
+ ),
+ )
+ print("✓ test_read_line_range passed")
+
+ _run(run())
+
+
+def test_read_absolute_path_rejected():
+ """Absolute paths are rejected (relative-only after refactor)."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ working = Path(tmp) / ".reme"
+ working.mkdir(parents=True, exist_ok=True)
+ target = _seed_md(working, "Abs.md", "x\n")
+ async with mock_reme_server() as (host, port):
+ result = await call_action(
+ "read",
+ host=host,
+ port=port,
+ path=str(target.resolve()),
+ )
+ if not (
+ isinstance(result, dict)
+ and result.get("success") is False
+ and "absolute" in str(result.get("answer", "")).lower()
+ ):
+ raise AssertionError(f"expected absolute-path rejection, got {result!r}")
+ print("✓ test_read_absolute_path_rejected passed")
+
+ _run(run())
+
+
+def test_read_non_md_rejected():
+ """Paths whose suffix is not `.md` are rejected."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ working = Path(tmp) / ".reme"
+ working.mkdir(parents=True, exist_ok=True)
+ async with mock_reme_server() as (host, port):
+ result = await call_action(
+ "read",
+ host=host,
+ port=port,
+ path="data/foo.txt",
+ )
+ if not (
+ isinstance(result, dict)
+ and result.get("success") is False
+ and "markdown" in str(result.get("answer", "")).lower()
+ ):
+ raise AssertionError(f"expected markdown-only rejection, got {result!r}")
+ print("✓ test_read_non_md_rejected passed")
+
+ _run(run())
+
+
+def test_read_missing_file():
+ """Reading a non-existent file should fail with a clear error."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ working = Path(tmp) / ".reme"
+ working.mkdir(parents=True, exist_ok=True)
+ async with mock_reme_server() as (host, port):
+ result = await call_action(
+ "read",
+ host=host,
+ port=port,
+ path="NotThere.md",
+ )
+ if not (
+ isinstance(result, dict)
+ and result.get("success") is False
+ and "does not exist" in str(result.get("answer", "")).lower()
+ ):
+ raise AssertionError(f"expected missing-file rejection, got {result!r}")
+ print("✓ test_read_missing_file passed")
+
+ _run(run())
+
+
+def test_read_start_after_end():
+ """start_line > end_line is invalid."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ working = Path(tmp) / ".reme"
+ working.mkdir(parents=True, exist_ok=True)
+ _seed_md(working, "Range.md", "a\nb\nc\n")
+ async with mock_reme_server() as (host, port):
+ result = await call_action(
+ "read",
+ host=host,
+ port=port,
+ path="Range.md",
+ start_line=3,
+ end_line=1,
+ )
+ if not (
+ isinstance(result, dict)
+ and result.get("success") is False
+ and "start_line" in str(result.get("answer", ""))
+ ):
+ raise AssertionError(f"expected start>end rejection, got {result!r}")
+ print("✓ test_read_start_after_end passed")
+
+ _run(run())
+
+
+def test_read_start_line_exceeds_total():
+ """start_line beyond total line count is invalid."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ working = Path(tmp) / ".reme"
+ working.mkdir(parents=True, exist_ok=True)
+ _seed_md(working, "Short.md", "only-one-line\n")
+ async with mock_reme_server() as (host, port):
+ result = await call_action(
+ "read",
+ host=host,
+ port=port,
+ path="Short.md",
+ start_line=99,
+ )
+ if not (
+ isinstance(result, dict)
+ and result.get("success") is False
+ and "exceeds" in str(result.get("answer", "")).lower()
+ ):
+ raise AssertionError(f"expected exceeds-length rejection, got {result!r}")
+ print("✓ test_read_start_line_exceeds_total passed")
+
+ _run(run())
+
+
+def test_read_truncation():
+ """A small max_bytes triggers truncation with a continuation notice."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ working = Path(tmp) / ".reme"
+ working.mkdir(parents=True, exist_ok=True)
+ body = "\n".join(f"line {i}" for i in range(200)) + "\n"
+ _seed_md(working, "Big.md", body)
+ async with mock_reme_server() as (host, port):
+ await call_and_check(
+ "read",
+ host=host,
+ port=port,
+ path="Big.md",
+ max_bytes=64,
+ validator=lambda r: (
+ isinstance(r, dict)
+ and r.get("success") is True
+ and "truncated" in str(r["answer"])
+ and "start_line=" in str(r["answer"])
+ ),
+ )
+ print("✓ test_read_truncation passed")
+
+ _run(run())
+
+
+def test_read_empty_path_rejected():
+ """An empty `path` should be rejected."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ working = Path(tmp) / ".reme"
+ working.mkdir(parents=True, exist_ok=True)
+ async with mock_reme_server() as (host, port):
+ result = await call_action("read", host=host, port=port, path="")
+ if not (
+ isinstance(result, dict)
+ and result.get("success") is False
+ and "required" in str(result.get("answer", "")).lower()
+ ):
+ raise AssertionError(f"expected `path` required rejection, got {result!r}")
+ print("✓ test_read_empty_path_rejected passed")
+
+ _run(run())
+
+
+# ---------------------------------------------------------------------------
+# Aggregate test: reuse one server instance for all read cases (faster).
+# ---------------------------------------------------------------------------
+
+
+def test_all_read_cases_one_server():
+ """Run multiple read scenarios against a single shared server for efficiency."""
+
+ async def run():
+ with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp):
+ working = Path(tmp) / ".reme"
+ working.mkdir(parents=True, exist_ok=True)
+ _seed_md(working, "Templates/Recipe.md", "# Recipe\nbody\n")
+ _seed_md(working, "Notes.md", "L1\nL2\nL3\n")
+ async with mock_reme_server() as (host, port):
+ await call_and_check(
+ "read",
+ host=host,
+ port=port,
+ path="Templates/Recipe.md",
+ validator=lambda r: r.get("success") is True and "# Recipe" in r["answer"],
+ )
+ await call_and_check(
+ "read",
+ host=host,
+ port=port,
+ path="Notes",
+ validator=lambda r: r.get("success") is True and "L1" in r["answer"],
+ )
+ await call_and_check(
+ "read",
+ host=host,
+ port=port,
+ path="Notes.md",
+ start_line=2,
+ end_line=2,
+ validator=lambda r: r.get("success") is True and r["answer"].strip() == "L2",
+ )
+ print("✓ test_all_read_cases_one_server passed")
+
+ _run(run())
+
+
+if __name__ == "__main__":
+ print("\n=== reme4 crud_md (read) E2E tests ===")
+ test_read_relative_path()
+ test_read_no_suffix_autoappends_md()
+ test_read_line_range()
+ test_read_absolute_path_rejected()
+ test_read_non_md_rejected()
+ test_read_missing_file()
+ test_read_start_after_end()
+ test_read_start_line_exceeds_total()
+ test_read_truncation()
+ test_read_empty_path_rejected()
+ test_all_read_cases_one_server()
+ print("\n所有测试通过!")
From 40feaa915099f926e8c821fe15f0cf54987cd028 Mon Sep 17 00:00:00 2001
From: jinliyl <6469360+jinliyl@users.noreply.github.com>
Date: Tue, 19 May 2026 16:20:09 +0800
Subject: [PATCH 16/16] fix(core): correct tool results directory naming (#246)
- Changed directory name from 'tool_result' to 'tool_results' in documentation
- Updated path variable assignment to use correct plural form 'tool_results'
- Ensured consistent directory naming throughout initialization logic
---
reme/reme_light.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/reme/reme_light.py b/reme/reme_light.py
index ca52a394..60f0a6c9 100644
--- a/reme/reme_light.py
+++ b/reme/reme_light.py
@@ -119,7 +119,7 @@ class ReMeLight(Application):
The following directory structure will be created:
- {working_dir}/ - Root working directory
- {working_dir}/memory/ - Memory storage files
- - {working_dir}/tool_result/ - Compacted tool result files
+ - {working_dir}/tool_results/ - Compacted tool result files
- {working_dir}/dialog/ - Raw conversation records
"""
# Initialize working directory structure
@@ -127,7 +127,7 @@ class ReMeLight(Application):
self.working_path.mkdir(parents=True, exist_ok=True)
self.memory_path = self.working_path / "memory"
self.memory_path.mkdir(parents=True, exist_ok=True)
- self.tool_result_path = self.working_path / "tool_result"
+ self.tool_result_path = self.working_path / "tool_results"
self.tool_result_path.mkdir(parents=True, exist_ok=True)
self.dialog_path = self.working_path / "dialog"
self.dialog_path.mkdir(parents=True, exist_ok=True)