Merge upstream main; resolve conflicts for seekdb, zvec, and hologres stores

.github/workflows/unittest.yml

@@ -0,0 +1,43 @@
+name: Tests ReMe
+
+on:
+  push:
+    branches: [main, master, dev, develop]
+  pull_request:
+    branches: [main, master, dev, develop]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  unit-tests:
+    name: Unit Tests - py${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
+          pip install -e ".[dev,core]"
+
+      - name: Run tests4 unit tests
+        run: |
+          pytest tests4/unittest \
+            -v \
+            --tb=long \
+            -s \
+            --log-cli-level=WARNING

.gitignore

@@ -45,4 +45,4 @@ meta_memory/*
 **/data/*.json
 *.db
 memories/*
-.reme/*
+.reme/*

README.md

@@ -33,7 +33,7 @@
 
 | Date       | Title                                                           |
 |------------|-----------------------------------------------------------------|
-| 2026-03-30 | [CoPaw Context Management Design](docs/copaw_context_design.md) |
+| 2026-03-30 | [Context Management Design](docs/copaw_context_design.md) |
 
 ---
 
@@ -53,7 +53,7 @@ ReMe achieves state-of-the-art results on the LoCoMo and HaluMem benchmarks; see
 
 <br>
 
-- **Personal assistant**: Provide long-term memory for agents like [CoPaw](https://github.com/agentscope-ai/CoPaw),
+- **Personal assistant**: Provide long-term memory for agents like [QwenPaw](https://github.com/agentscope-ai/CoPaw),
   remembering user preferences and conversation history.
 - **Coding assistant**: Record code style preferences and project context, maintaining a consistent development
   experience across sessions.
@@ -73,7 +73,7 @@ ReMe achieves state-of-the-art results on the LoCoMo and HaluMem benchmarks; see
 > Memory as files, files as memory.
 
 Treat **memory as files** — readable, editable, and copyable.
-[CoPaw](https://github.com/agentscope-ai/CoPaw) integrates long-term memory and context management by inheriting from
+[QwenPaw](https://github.com/agentscope-ai/CoPaw) integrates long-term memory and context management by inheriting from
 `ReMeLight`.
 
 | Traditional memory system | File-based ReMe      |
@@ -245,7 +245,7 @@ flowchart TD
 
 ---
 
-[CoPaw MemoryManager](https://github.com/agentscope-ai/CoPaw/blob/main/src/copaw/agents/memory/reme_light_memory_manager.py)
+[MemoryManager](https://github.com/agentscope-ai/CoPaw/blob/main/src/copaw/agents/memory/reme_light_memory_manager.py)
 inherits `ReMeLight` and integrates its memory capabilities into the agent reasoning loop:
 
 ```mermaid
@@ -506,7 +506,7 @@ async def main():
             "dimensions": 1024,
         },
         default_vector_store_config={
-            "backend": "local",  # Supports local/chroma/qdrant/elasticsearch/obvec
+            "backend": "local",  # Supports local/chroma/qdrant/elasticsearch/obvec/zvec/hologres
         },
     )
     await reme.start()
@@ -671,7 +671,7 @@ For more details on how to reproduce the experiments, see [quickstart.md](benchm
 - **Need a new feature?** Open a feature request; we’ll evolve ReMe together with the community.
 - **Code contributions**: All forms of contributions are welcome. Please see
   the [contribution guide](docs/contribution.md).
-- **Acknowledgements**: We thank excellent open-source projects such as OpenClaw, Mem0, MemU, and CoPaw for their
+- **Acknowledgements**: We thank excellent open-source projects such as OpenClaw, Mem0, MemU, and QwenPaw for their
   inspiration and support.
 
 ### Contributors

README_ZH.md

@@ -486,7 +486,7 @@ async def main():
             "dimensions": 1024,
         },
         default_vector_store_config={
-            "backend": "local",  # 支持 local/chroma/qdrant/elasticsearch/obvec
+            "backend": "local",  # 支持 local/chroma/qdrant/elasticsearch/obvec/zvec
         },
     )
     await reme.start()

benchmark/halumem/quickstart.md

@@ -14,17 +14,30 @@ conda activate ./reme-env
 pip install .
 ```
 
-### 2. Clone the Repository
+### 2. Download the Dataset
 ```bash
 cd ./benchmark/halumem
-git clone https://github.com/MemTensor/HaluMem.git
+mkdir -p data
+curl -L "https://huggingface.co/datasets/IAAR-Shanghai/HaluMem/resolve/main/HaluMem-Medium.jsonl?download=true" -o data/HaluMem-Medium.jsonl
+curl -L "https://huggingface.co/datasets/IAAR-Shanghai/HaluMem/resolve/main/HaluMem-Long.jsonl?download=true" -o data/HaluMem-Long.jsonl
+```
+
+Dataset page:
+https://huggingface.co/datasets/IAAR-Shanghai/HaluMem/tree/main
+
+If the official source is slow or inaccessible in mainland China, you can use a mirror:
+```bash
+cd ./benchmark/halumem
+mkdir -p data
+curl -L "https://hf-mirror.com/datasets/IAAR-Shanghai/HaluMem/resolve/main/HaluMem-Medium.jsonl?download=true" -o data/HaluMem-Medium.jsonl
+curl -L "https://hf-mirror.com/datasets/IAAR-Shanghai/HaluMem/resolve/main/HaluMem-Long.jsonl?download=true" -o data/HaluMem-Long.jsonl
 ```
 
 ### 3. Run Experiments
 Launch the ReMe service to enable memory library functionality:
 ```bash
 clear && python benchmark/halumem/eval_reme.py \
-    --data_path benchmark/halumem/HaluMem/data/HaluMem-Medium.jsonl \
+    --data_path benchmark/halumem/data/HaluMem-Medium.jsonl \
     --reme_model_name gpt-4o-mini-2024-07-18 \
     --eval_model_name gpt-4o-mini-2024-07-18 \
     --batch_size 40 \

benchmark/locomo/eval_reme.py

@@ -643,7 +643,7 @@ class LocomoEvaluator:
             },
             "personal_retriever": {
                 "prompt_dict": {
-                    "user_message": self.retriever_prompt,
+                    "user_message_s2": self.retriever_prompt,
                 },
                 "params": {
                     "return_memory_nodes": True,

benchmark/locomo/eval_reme.yaml

@@ -132,7 +132,7 @@ user_message_retrieve: |
   You are a Memory Retrieval Agent specialized in retrieving {memory_type} memories about {memory_target}.
 
   ## User Profile
-  {user_profile}
+  {profiles}
 
   ## User Question
   {context}

docs/index.md

@@ -139,7 +139,7 @@ response = requests.post("http://localhost:8002/retrieve_task_memory", json={
 ## 📚 Resources
 
 - **[Installation Guide](installation.md)**, **[Quick Start](quick_start.md)**: Get started quickly with practical examples
-- **[Vector Storage Setup](vector_store_api_guide.md)**: Configure local, Elasticsearch, Qdrant, ChromaDB, or ObVec (OceanBase / seekdb via pyobvector) storage and usage
+- **[Vector Storage Setup](vector_store_api_guide.md)**: Configure local, Elasticsearch, Qdrant, ChromaDB, ObVec (OceanBase / seekdb via pyobvector) or Hologres storage and usage
 - **[MCP Guide](mcp_quick_start.md)**: Create MCP services
 - **[Personal Memory](personal_memory/personal_memory.md)**, **[Task Memory](task_memory/task_memory.md)** & **[Tool Memory](tool_memory/tool_memory.md)**: Operators used in personal memory, task memory and tool memory. You can modify the config to customize the pipelines.
 - **[Example Collection](./cookbook/appworld/quickstart.md)**: Real use cases and best practices

docs/vector_store_api_guide.md

@@ -34,6 +34,8 @@ FlowLLM provides multiple Vector Store implementations tailored to different use
 - **ChromaVectorStore** ([source code](https://github.com/flowllm-ai/flowllm/blob/main/flowllm/core/vector_store/chroma_vector_store.py)): Based on ChromaDB, providing persistent storage and metadata filtering capabilities.
 - **EsVectorStore** ([source code](https://github.com/flowllm-ai/flowllm/blob/main/flowllm/core/vector_store/es_vector_store.py)): Built on Elasticsearch, enabling powerful combined full-text and vector search functionalities.
 - **ObVecVectorStore** ([source code](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/obvec_vector_store.py)): Uses [pyobvector](https://pypi.org/project/pyobvector/) against **OceanBase** or **seekdb** (MySQL-compatible wire protocol). Suitable when you already run OceanBase/seekdb or need a SQL-native vector table with HNSW-style ANN search and JSON metadata filters.
+- **HologresVectorStore** ([source code](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/hologres_store.py)): Uses [asyncpg](https://pypi.org/project/asyncpg/) against **Hologres** (PostgreSQL-compatible). Leverages native `float4[]` vector storage with built-in HGraph index for approximate nearest neighbor search. Suitable when you already run Hologres or need high-performance vector search with JSONB metadata filtering in a PostgreSQL-compatible environment.
+- **ZvecVectorStore** ([source code](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/zvec_vector_store.py)): Built on zvec, a high-performance local vector database with strong-schema support and HNSW indexing. Suitable for single-machine deployments requiring fast vector search.
 
 All Vector Store implementations inherit from **BaseVectorStore** ([source code](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/base_vector_store.py)) in ReMe, ensuring a consistent interface specification.
 
@@ -130,6 +132,25 @@ docker run -d --name reme_seekdb -p 2881:2881 -e ROOT_PASSWORD=<your_root_passwo
 ```shell
 OBVEC_PASSWORD=<your_root_password> python tests/test_vector_store.py --obvec
 ```
+### ZvecVectorStore Configuration
+
+- **db_path**: Local storage path for persistent mode (required).
+- **dimension**: Dimensionality of the embedding vectors (default: `1024`).
+- **distance**: Distance metric — supports `cosine`, `l2`, `ip` (default: `cosine`).
+
+### HologresVectorStore Configuration
+
+- **host**: Hologres host address (default: `localhost`).
+- **port**: Hologres port (default: `80`).
+- **database**: Database name (default: `postgres`).
+- **user**: Database user (default: `postgres`).
+- **password**: Database password.
+- **schema**: PostgreSQL schema name (default: `public`).
+- **min_size**: Minimum connections in pool (default: `1`).
+- **max_size**: Maximum connections in pool (default: `10`).
+- **dsn**: Full DSN connection string. When provided, overrides `host`, `port`, `database`, `user`, and `password`.
+- **distance_method**: Distance method for the HGraph index: `Cosine`, `InnerProduct`, or `Euclidean` (default: `Cosine`).
+- **collection_name**: Table name for the collection (from `VectorStoreConfig`, default `reme`).
 
 ## Configuration File Examples
 
@@ -151,7 +172,7 @@ vector_store.default.params.<param_name>=<param_value>
 
 ### Configuration Field Descriptions
 
-- **`backend`** (required): Vector store backend type. Options: `local`, `memory`, `chroma`, `qdrant`, `elasticsearch`, `obvec`.
+- **`backend`** (required): Vector store backend type. Options: `local`, `memory`, `chroma`, `qdrant`, `elasticsearch`, `obvec`, `zvec`, `hologres`.
 - **`embedding_model`** (required): Name of the embedding model configuration, referencing the `embedding_model` section.
 - **`params`** (optional): Dictionary of backend-specific parameters passed to the vector store constructor.
 
@@ -347,6 +368,60 @@ vector_stores.default.password=your-root-password
 
 ReMe service YAML uses the key `vector_stores` (plural); CLI overrides use the same nested paths.
 
+#### 7. HologresVectorStore Configuration
+
+**Implementation**: [`reme/core/vector_store/hologres_store.py`](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/hologres_store.py)
+
+**Example (Hologres instance)**:
+
+```yaml
+vector_stores:
+  default:
+    backend: hologres
+    embedding_model: default
+    collection_name: reme
+    host: "your-hologres-host"
+    port: 80
+    database: "postgres"
+    user: "postgres"
+    password: "your-password"
+    schema: "public"
+    distance_method: "Cosine"
+```
+
+```shell
+vector_stores.default.backend=hologres
+vector_stores.default.host=your-hologres-host
+vector_stores.default.port=80
+vector_stores.default.user=postgres
+vector_stores.default.password=your-password
+vector_stores.default.database=postgres
+```
+
+#### 8. ZvecVectorStore Configuration
+
+Persistent local storage based on zvec with HNSW indexing and strong-schema support.
+
+**Implementation**: [`reme/core/vector_store/zvec_vector_store.py`](https://github.com/agentscope-ai/ReMe/blob/main/reme/core/vector_store/zvec_vector_store.py)
+
+```yaml
+vector_store:
+  default:
+    backend: zvec
+    embedding_model: default
+    params:
+      db_path: "./zvec_vector_store"  # Local storage path (required)
+      dimension: 1024                 # Vector dimension (optional; default: 1024)
+      distance: "cosine"              # Distance metric (optional; default: cosine; options: cosine, l2, ip)
+```
+
+```shell
+vector_store.default.backend=zvec
+vector_store.default.params.db_path=./zvec_vector_store
+vector_store.default.params.dimension=1024
+vector_store.default.params.distance=cosine
+```
+
 ### Complete Configuration Example
 
 Below is a complete `default.yaml` example including both embedding model and vector store configurations:
@@ -404,9 +479,11 @@ Two types of metadata filtering are supported:
 
 - **Development & Testing**: Use MemoryVectorStore or LocalVectorStore—no additional services required.
 - **Small-Scale Applications**: Use LocalVectorStore or ChromaVectorStore for simplicity and ease of use.
-- **Production Environments**: Use QdrantVectorStore, EsVectorStore, or ObVecVectorStore (OceanBase/seekdb) for high performance and scalability, depending on your existing infrastructure.
+- **Production Environments**: Use QdrantVectorStore, EsVectorStore, ObVecVectorStore (OceanBase/seekdb), or HologresVectorStore for high performance and scalability, depending on your existing infrastructure.
+- **High-Performance Local Search**: Use ZvecVectorStore for single-machine deployments requiring fast HNSW-based vector search with local persistence.
 - **Hybrid Search**: Use EsVectorStore to combine vector search with full-text search capabilities.
 - **OceanBase / seekdb**: Use ObVecVectorStore when you standardize on pyobvector and SQL-accessible vector tables.
+- **Hologres**: Use HologresVectorStore when you run Hologres and need native HGraph-indexed vector search with PostgreSQL-compatible SQL and JSONB metadata filtering.
 
 ## Important Notes
 

docs4/reme_design.md

@@ -0,0 +1,183 @@
+# 快速测试
+
+```bash
+# 终端 A：启动服务
+reme4 start
+
+# 终端 B：调用 version 验证服务可用
+reme4 version
+# 预期输出：✅ ReMe v{__version__}
+```
+
+# 基础Job
+
+@jinli
+
+入口：`reme4/reme.py::main()` → `parse_args(*sys.argv[1:])` 解析首个位置参数为 `action`，后续 `key=value` 解析为 kwargs（支持
+`service.port=8080` 的 dot notation；自动剥离 `--` / `-` 前缀；值会做 bool / int / float / JSON 转换）。
+
+调用模式：
+
+- `start`：本地启动 `ReMe(Application)` 服务（不经过 client）
+- `find_reme`：本地探测正在运行的 reme，不调用服务
+- `list`：在 client 端拦截，不转发到服务端，直接返回 action 目录
+- 其他 action：通过 `call_server(action, **kwargs)` → `R.get(ComponentEnum.CLIENT, backend)` 实例化客户端并流式打印（任意未列出的
+  step register name 都按本规则透传）
+
+通用可选参数 `backend:str=http`（取值 `http` / `mcp`，对应 `reme4/components/client/{http_client,mcp_client}.py` 中
+`@R.register` 注册名）；服务端默认 host/port 见 `reme4/constants.py`，可由 `start` 端通过 `service.host=` / `service.port=`
+覆盖。
+
+说明：📥 输入参数 ｜ 📤 输出 ｜ ⭐ 必填 ｜ 🎚️ 默认值 ｜ 🛠️ 内部行为 ｜ 📊 metadata
+
+| 分类         | 指令 (register name)                               | 入口                                                          | 参数 & 行为                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+|------------|--------------------------------------------------|-------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 🚀 本地      | 🟢 `start`                                       | `reme.py:30` → `ReMe(**kwargs).run_app()`                   | 📥 可选 `config=<name\|path>`（默认加载 `reme4/config/default.yaml`，`.yaml/.yml/.json` 都支持，含 `${ENV:-default}` 占位符）｜ 可选 `service.host=` / `service.port=` 等任意 dot-notation 覆盖 ｜ 🛠️ 流程：`load_env()` → `resolve_app_config(**kwargs)` deep merge → `precheck_start(svc)`（`utils/service_utils.py:72`：目标 host:port 已有 reme → 打印 `reme already running ...` 直接返回；端口被其他进程占用 → stderr 提示 `port {port} occupied. Start on another port: reme4 start service.port=<other_port>` 并 `sys.exit(1)`）→ 启动服务                                                                                                                                                                                        |
+| 🚀 本地      | 🧭 `find_reme`                                   | `reme.py:36` → `utils/service_utils.py:89`                  | 📥 无 ｜ 📤 发现服务则 stdout 打印 `HOST={host} PORT={port} PID={pid or 'unknown'}`；未发现则 stderr 提示 `reme not started. Try: reme start` 并 `sys.exit(1)` ｜ 🛠️ 流程：先探 `REME_DEFAULT_HOST:REME_DEFAULT_PORT`（`health_check` 命中算 `reme`），再 `pgrep -af "reme.* start"` 扫描其他端口                                                                                                                                                                                                                                                                                                                                                                                                                  |
+| 🛰️ 客户端    | 📜 `list`                                        | `components/client/base_client.py:36`                       | 📥 无 ｜ 📤 服务端可用 action 目录（JSON，`indent=2 ensure_ascii=False`）｜ 🛠️ 在 `BaseClient.__call__` 中拦截，不进入 `_execute`，直接调用 `list_actions()`（HTTP/MCP backend 各自实现）                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| 🌐 通用 step | 🆘 `help` (`help_step`)                          | `call_server("help")`                                       | 📥 无 ｜ 📤 `answer` 一行一个 job：`🛠️ \`{name}\` — {description} 📥 {params}`，参数渲染为 `name:type*`(必填) / `name:type={default}` / `name:type` ｜ 📊 `metadata.job_count` ｜ 🛠️ 自动跳过名为 `help` 的 job                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+| 🌐 通用 step | 🩺 `health_check` (`health_check_step`)          | `call_server("health_check")`                               | 📥 无 ｜ 📤 `answer = "✅/❌ ReMe v{version} - healthy/unhealthy"` ｜ 📊 `metadata.health = {version, healthy, components}` ｜ 🧩 覆盖组件：`embedding_model`(🟢 is_started/is_healthy/model_name/dimensions/cache_size/memory) · `file_graph`(🕸️ n_nodes/n_edges/n_virtual\|n_pending/memory) · `file_store`(📦 n_chunks/n_chunks_with_embedding/memory) · `file_watcher`(👀 background_running/watch_paths) · `keyword_index`(🔤 n_docs/vocab_size/memory) ｜ 🛠️ deep sizeof（含 numpy.nbytes），未启动 / 后台未跑 / embedding 不健康 → ❌                                                                                                                                                             |
+| 🌐 通用 step | 🏷️ `version` (`version_step`)                   | `call_server("version")`                                    | 📥 无 ｜ 📤 `answer = reme4.__version__` ｜ 📊 `metadata.version`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
+| 🌐 通用 step | 🔄 `reindex` (`reindex_step`)                    | `call_server("reindex")`                                    | 📥 无 ｜ 📤 `answer = "🔄 Reindexed {added} file(s)"` ｜ 📊 `metadata.counts = {added, ...}` ｜ 🛠️ 流程：`file_watcher.close()` → `file_store.clear()` → `file_watcher.update_store()` → `file_watcher.start()`（finally 保证重启）                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| 🔎 search  | 🔍 `search` (`search_step`)                      | `call_server("search", query=…, …)`                         | 📥 `query:str` ⭐ ｜ 🎚️ `limit:int=5`(>0) ｜ 🎚️ `min_score:float=0.0` ｜ ⚖️ `vector_weight:float=0.7` ∈[0,1]（keyword 权 = 1-vw）｜ 🔀 `candidate_multiplier:float=3.0`（candidates = min(200, limit×mult)）｜ 🔗 `expand_links:bool=True` ｜ 🔢 `max_links_per_direction:int=10` ｜ 🎚️ `search_filter:dict={}` ｜ 📤 `answer` 每命中一行 `path:start-end [score=… vector=… keyword=…] text` + 缩进的 `→ outlinks (n)` / `← inlinks (n)` + `via predicate=… anchor=#…` ｜ 📊 `metadata.results` / `metadata.link_expansion` / `metadata.counts={vector,keyword,returned,hybrid}` ｜ 🛠️ 并行 `vector_search` + `keyword_search` → RRF 融合（K=60，按 chunk.id 合并）→ `min_score` 过滤 → `limit` 截断 → 邻居 meta 注入 |
+| 🧪 demo    | 🪄 `demo_echo` (`demo_echo_step1` + `step2`)     | `call_server("demo_echo", query=…, min_score=…)`            | 📥 `query:str=""` ｜ 🎚️ `min_score:float=0.5` ｜ 🛠️ step1：`processed_query = query.strip().lower()`，`adjusted_min_score = min_score * 0.9`，写回 context ｜ 📤 step2：`answer = "echo: {processed_query} (min_score={adjusted_min_score})"` ｜ 📊 `metadata = {step, query, min_score, processed_query, adjusted_min_score}`                                                                                                                                                                                                                                                                                                                                                          |
+| 🌊 demo    | 🌊 `stream_demo` (`stream_demo_step1` + `step2`) | `call_server("stream_demo", query=…, repeat=…, interval=…)` | 📥 `query:str=""` ｜ 🎚️ `repeat:int=10` ｜ 🎚️ `interval:float=0.1`（秒/字符）｜ 🛠️ step1：`stream_text = query * repeat` 写回 context ｜ 📤 step2：按字符 `add_stream_string(ch, ChunkEnum.CONTENT)` 流式输出，`asyncio.sleep(interval)` 节流                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+| 📂 crud    | 📖 `read` (`read_step`)                          | `call_server("read", path=…, …)`                            | 📥 `path:str` ⭐（**完整相对路径**，相对于 `working_dir`；绝对路径会被拒绝；非 `.md` 后缀拒绝）｜ 🎚️ `start_line:int=null`（1-based, 含端点）｜ 🎚️ `end_line:int=null`（1-based, 含端点）｜ 🎚️ `max_bytes:int=51200`（截断阈值）｜ 📤 `answer = 选中的行内容`，超过 `max_bytes` 时附加 `--- TRUNCATED ---` 续读指引（`start_line=…`）｜ 📊 `metadata.path` / `metadata.total_lines`（出错路径才会附带）｜ 🛠️ 流程：`BaseStep.resolve_path(raw, require_md=True)` → `aiofiles.os.stat` → `read_file_safe`（utf-8-sig BOM 容忍、UnicodeDecodeError fallback `errors=ignore`）→ `split("\n")` 切片 `[s-1:e]` → `truncate_text_output` 按字节截断保行                                                                                              |
+
+使用示例：
+
+```bash
+# 启动（默认 default.yaml）
+reme4 start
+
+# 指定 config 与服务端口
+reme4 start config=paw.yaml service.port=8181
+
+# 查找在跑的 reme
+reme4 find_reme
+# HOST=127.0.0.1 PORT=8000 PID=12345
+
+# 列出所有可用 action（client 端处理，不转服务端）
+reme4 list
+
+# 转发到服务端的 step：所有 key=value 透传为 step kwargs
+reme4 help
+reme4 health_check
+reme4 version
+reme4 reindex
+reme4 search query="latency 问题" limit=10 min_score=0.2 vector_weight=0.6
+
+# 读取 working_dir 下的 markdown（完整相对路径；无后缀自动补 .md；可按行切片或限制字节）
+reme4 read path=Templates/Recipe.md
+reme4 read path=Notes start_line=1 end_line=20
+reme4 read path=Big.md max_bytes=4096
+
+# 通过 MCP backend 调用
+reme4 search query="..." backend=mcp
+```
+
+@sen
+| tags | stat | 返回特定tag信息 |
+| tags | list | 返回所有tag列表 |
+| crud | upload/download | 其他文件 |
+| file | stat | path |
+| file | list | path |
+| property | property:read | |
+| property | property:update | path="My Note" status=done xx=xxx |
+| property | property:delete | keys="[xxxx, xxxx]"                                               |
+| graph | traverse | path="My Note"  directtion=forward/backward depth=1 predicat=xxx |
+
+@wangce
+| crud | create | path="New Note" content="# Hello" title="xxx" tags="[]" status="" |
+| crud | read | path="Templates/Recipe.md"                                        |
+| crud | edit | path="Templates/Recipe.md" old="xxx" new="xxx"                    |
+| crud | append | path="My Note" content="New line"                                 |
+| crud | prepend | path="My Note" content="New line"                                 |
+| crud | delete | path="My Note
+| daily:crud | daily:xxx | 与 crud 参数保持一致 |
+
+# 日记类型
+
+| 类型        | 路径                                            | 说明                          |
+|-----------|-----------------------------------------------|-----------------------------|
+| daily     | {daily}/xxxx-mm-dd.md + xxxx-mm-dd/{event}.md | 按日期归档的原始信息记录                |
+| topic     | topic/{topic:-personal(agent)}/{xxxx}.md      | 按主题聚类的二次加工内容                |
+| proactive | todo                                          | 基于 daily / topic 思考后主动推送的消息 |
+
+# 生成Job
+
+| 任务                      | 输入            | 输出                                            | 触发时机                        | 说明                                                   |
+|-------------------------|---------------|-----------------------------------------------|-----------------------------|------------------------------------------------------|
+| 日记summary @sen @wangce  | msg           | {daily}/xxxx-mm-dd.md + xxxx-mm-dd/{event}.md | freq (every_n_turn、compact) | 把 msg 的信息写入 daily 目录                                 |
+| 主题dream  + 生成链接 @sen    | daily/xxx     | knowledge/xxx                                 | /dream                      | 把 daily 目录的内容按主题聚类合并到 topic 目录, 主动在文档中建立 [[link]] 关联 |
+| 主动proactive     @wangce | daily / topic | proactive_query                               | pre_query                   | 思考 daily / topic 信息，主动决定推送给用户的消息                     |
+
+2. file_parser
+   a. 抽象基类 parse: @jinli
+   ⅰ. 输入是path：相对路径
+   ⅱ. 输出是FileMetadata & list[FileChunks] & list[FileEdge]
+   b. default parser 兼容老方案 @jinli
+   ⅰ. 带overlap的chunking策略 ，不输出FileEdge
+   c. markdown parser @sen
+   ⅰ. 根据markdown ast做chunk，不需要overlap
+   ⅱ. 增加一个索引的chunk chunk_type @锦鲤 file_chunk_type content/index
+   ⅲ. 增加link的正则解析：predicate:: [[path#anchor]]
+3. file_store @sen
+   a. 抽象存储：
+   ⅰ. filenode = file + path + st_mtime + metadata + list[FileEdge]
+   ⅱ. graph=dict[str, filenode] 内存+json
+   ⅲ. list[FileChunk] 存db
+   b. 抽象基类
+   ⅰ. graph：fellow dict的操作 update/get/set
+   ⅱ. chunks dict[str, list[chunk]]
+    1. delete_chunks_by_path
+    2. update_chunks_by_path
+    3. list_chunks_by_path
+    4. vector_search/keyword_search
+       ⅲ. 手写一个bm25检索
+       ⅳ. 【核心】检索机制 vector bm25 graph 如何进行融合
+4. file_watcher @jinli
+   a. 抽象基类
+   ⅰ. on_start:
+    1. file_store 的start 在前，加载graph，file_watcher在后，递归扫描目录
+       a. 通过ms_time对比graph，on_change 进行改动
+       ⅱ. on_change:
+    1. 更新/增加:
+       a. delete_chunks_by_path 更新数据库
+       b. upate_chunks_by_path 更新数据库
+       c. 更新graph
+    2. 删除
+       a. delete_chunks_by_path 更新数据库
+
+MemorySchema
+
+1. markdown文件结构 @sen
+   a. formatter：
+   ⅰ. title
+   ⅱ. desc
+   ⅲ. tags
+   ⅳ.
+2. memory文件结构目录
+   a. MEMORY.md
+   b. msg/files -> daily/YYYYMMDD/YYYYMMDD.md + xxxx.md
+   ⅰ. YYYYMMDD.md
+    1. xxx -> xxxx.md
+    2. xxx -> xxxd.md
+       ⅱ.
+       c. daily -> topic/topic_l1/topic_l1.md + xxx.md + topic_l2
+       d. proactive
+
+steps:
+
+1. 治理（算法+LLM）：
+   a. 节点关联P0：现有的链接做补充，挖掘新的LLM的link
+   ⅰ. /Users/yuli/workspace/ReMe/reme2/component/edge_extractor/llm_edge_extractor.py
+   ⅱ. 移动到steps
+   b. 节点整合/节点拆分/节点归档
+   c. 健康度检查
+2. retrieve 调用store的检索
+3. 原子steps：reme edit
+4. 组合steps：总结：
+   a. - freq (every_n_turn、compact) -> daily_summarizer
+   b. topic (/dream ) -> topic_summarizer(daily_xx -> topic_xx)
+   c. proactive -> proactive_summarizer(personal_xxx -> proactive_query - pre_query

docs4/todo.md

@@ -0,0 +1,7 @@
+1. 完善mcp_servers config
+2. 完善mcp/http的服务测试
+3. [PosixPath('.reme')]
+4. error
+5. meta信息存在一个地方
+6. 测试一个完整的Service client的框架，测试各种命令
+7. config 默认改成default

pyproject.toml

@@ -33,9 +33,7 @@ classifiers = [
 keywords = ["llm", "memory", "experience", "memoryscope", "ai", "mcp", "http", "reme", "personal"]
 
 dependencies = [
-    "sqlite-vec>=0.1.6",
-    "prompt_toolkit>=3.0.52",
-    "rich>=14.2.0",
+    "aiofiles>=24.1.0",
     "asyncpg>=0.31.0",
     "chromadb>=1.3.5",
     "pyseekdb>=1.2.0",
@@ -44,16 +42,22 @@ dependencies = [
     "fastapi>=0.121.3",
     "fastmcp>=2.14.1",
     "httpx>=0.28.1",
+    "jieba>=0.42.1",
     "loguru>=0.7.3",
     "mcp>=1.25.0",
+    "networkx>=3.4",
     "numpy>=2.2.6",
     "openai>=2.8.1",
     "pandas>=2.3.3",
+    "prompt_toolkit>=3.0.52",
     "pydantic>=2.12.4",
     "pyobvector>=0.1.20",
+    "pyyaml>=6.0.3",
+    "qdrant-client>=1.16.0",
+    "rich>=14.2.0",
+    "sqlite-vec>=0.1.6",
     # pyobvector imports Expression from sqlglot; removed from sqlglot 30+ top-level API
     "sqlglot>=25,<30",
-    "qdrant-client>=1.16.0",
     "tavily-python>=0.7.13",
     "tiktoken>=0.12.0",
     "tqdm>=4.67.1",
@@ -61,6 +65,8 @@ dependencies = [
     "uvicorn>=0.40.0",
     "watchfiles>=1.1.1",
     "pyyaml>=6.0.3",
+    "mistletoe",
+    "neo4j",
 ]
 
 [project.optional-dependencies]
@@ -76,6 +82,8 @@ dev = [
     "furo",
     "sphinxcontrib-mermaid",
     "pre-commit",
+    "pytest>=8.0",
+    "pytest-asyncio>=0.23",
 ]
 
 full = [
@@ -87,13 +95,17 @@ litellm = [
 ]
 
 light = [
-    "agentscope==1.0.18",
+    "agentscope==1.0.19",
     "flowllm[reme]>=0.2.0.10",
 ]
 
+core = [
+    "agentscope==1.0.19",
+]
+
 [tool.setuptools.packages.find]
 where = ["."]
-include = ["reme_ai*", "reme*"]
+include = ["reme_ai*", "reme*", "reme4*"]
 exclude = ["test*", "cookbook*", "doc*", "library*", "dist*"]
 
 [tool.setuptools.package-data]
@@ -109,6 +121,12 @@ reme = [
     "**/*.json",
 ]
 
+reme4 = [
+    "**/*.yaml",
+    "**/*.py",
+    "**/*.json",
+]
+
 [tool.setuptools.dynamic]
 version = { attr = "reme.__version__" }
 
@@ -121,6 +139,7 @@ Repository = "https://github.com/agentscope-ai/ReMe"
 reme = "reme_ai.main:main"
 reme2 = "reme.reme:main"
 remecli = "reme.reme_cli:main"
+reme4 = "reme4.reme:main"
 
 [tool.pytest.ini_options]
 asyncio_default_fixture_loop_scope = "function"

reme/__init__.py

@@ -6,7 +6,7 @@ from . import extension
 from . import memory
 from .reme import ReMe
 
-__version__ = "0.3.1.8"
+__version__ = "0.3.1.9"
 
 __all__ = [
     "config",

reme/core/application.py

@@ -44,6 +44,7 @@ class Application:
         config_path: str | None = None,
         enable_logo: bool = True,
         log_to_console: bool = True,
+        log_to_file: bool = True,
         enable_load_env: bool = True,
         parser: type[PydanticConfigParser] | None = None,
         default_as_llm_config: dict | None = None,
@@ -73,6 +74,7 @@ class Application:
             config_path=config_path,
             enable_logo=enable_logo,
             log_to_console=log_to_console,
+            log_to_file=log_to_file,
             default_as_llm_config=default_as_llm_config,
             default_as_llm_formatter_config=default_as_llm_formatter_config,
             default_llm_config=default_llm_config,
@@ -144,7 +146,10 @@ class Application:
             logger.warning("Application has already started.")
             return self
 
-        init_logger(log_to_console=self.service_config.log_to_console)
+        init_logger(
+            log_to_console=self.service_config.log_to_console,
+            log_to_file=self.service_config.log_to_file,
+        )
         logger.info(f"Init ReMe with config: {self.service_config.model_dump_json()}")
 
         working_path = Path(self.service_config.working_dir)

reme/core/file_store/__init__.py

@@ -9,6 +9,7 @@ from .base_file_store import BaseFileStore
 from .chroma_file_store import ChromaFileStore
 from .local_file_store import LocalFileStore
 from .sqlite_file_store import SqliteFileStore
+from .zvec_file_store import ZvecFileStore
 from ..registry_factory import R
 
 __all__ = [
@@ -16,11 +17,13 @@ __all__ = [
     "ChromaFileStore",
     "LocalFileStore",
     "SqliteFileStore",
+    "ZvecFileStore",
 ]
 
 R.file_stores.register("sqlite")(SqliteFileStore)
 R.file_stores.register("chroma")(ChromaFileStore)
 R.file_stores.register("local")(LocalFileStore)
+R.file_stores.register("zvec")(ZvecFileStore)
 
 try:
     from .seekdb_file_store import SeekdbFileStore

reme/core/file_store/zvec_file_store.py

@@ -0,0 +1,573 @@
+"""Zvec storage backend for file store."""
+
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+from typing import Any
+
+from .base_file_store import BaseFileStore
+from ..enumeration import MemorySource
+from ..schema import FileMetadata, MemoryChunk, MemorySearchResult
+from ..utils import get_logger
+
+logger = get_logger()
+
+_ZVEC_IMPORT_ERROR: Exception | None = None
+
+try:
+    import zvec  # type: ignore[import-untyped]
+    from zvec import (
+        CollectionOption,
+        CollectionSchema,
+        DataType,
+        Doc,
+        FieldSchema,
+        HnswIndexParam,
+        InvertIndexParam,
+        VectorQuery,
+        VectorSchema,
+    )
+    from zvec.typing import MetricType
+except Exception as e:
+    _ZVEC_IMPORT_ERROR = e
+    zvec = None  # type: ignore[assignment]
+
+
+# zvec max topk (will be lifted to 100,000 in zvec v0.3.2+)
+_ZVEC_MAX_TOPK = 1024
+
+# Default vector field name
+_DEFAULT_VECTOR_FIELD = "embedding"
+
+
+def _escape(value: str) -> str:
+    """Escape a string value for zvec filter expressions."""
+    return value.replace("'", "\\'")
+
+
+def _build_file_store_schema(name: str, dimension: int) -> CollectionSchema:
+    """Build a zvec CollectionSchema for file store chunks."""
+    return CollectionSchema(
+        name=name,
+        fields=[
+            FieldSchema("content", DataType.STRING, nullable=True, index_param=InvertIndexParam()),
+            FieldSchema("path", DataType.STRING, nullable=True, index_param=InvertIndexParam()),
+            FieldSchema("source", DataType.STRING, nullable=True, index_param=InvertIndexParam()),
+            FieldSchema("start_line", DataType.INT64, nullable=True),
+            FieldSchema("end_line", DataType.INT64, nullable=True),
+            FieldSchema("hash", DataType.STRING, nullable=True),
+            FieldSchema("updated_at", DataType.INT64, nullable=True),
+            FieldSchema("file_metadata", DataType.STRING, nullable=True),
+        ],
+        vectors=[
+            VectorSchema(
+                name=_DEFAULT_VECTOR_FIELD,
+                data_type=DataType.VECTOR_FP32,
+                dimension=dimension,
+                index_param=HnswIndexParam(metric_type=MetricType.COSINE),
+            ),
+        ],
+    )
+
+
+def _chunk_to_doc(chunk: MemoryChunk, file_meta_json: str = "{}") -> Doc:
+    """Convert a MemoryChunk to a zvec Doc."""
+    fields: dict[str, Any] = {
+        "content": chunk.text,
+        "path": chunk.path,
+        "source": chunk.source.value if chunk.source else "",
+        "start_line": chunk.start_line,
+        "end_line": chunk.end_line,
+        "hash": chunk.hash,
+        "updated_at": int(time.time() * 1000),
+        "file_metadata": file_meta_json,
+    }
+    vectors: dict[str, Any] = {}
+    if chunk.embedding is not None:
+        vectors[_DEFAULT_VECTOR_FIELD] = chunk.embedding
+    return Doc(id=chunk.id, fields=fields, vectors=vectors)
+
+
+def _doc_to_chunk(doc: Doc) -> MemoryChunk:
+    """Convert a zvec Doc to a MemoryChunk."""
+    raw_vector = doc.vector(_DEFAULT_VECTOR_FIELD)
+    vector = raw_vector if isinstance(raw_vector, list) and len(raw_vector) > 0 else None
+    return MemoryChunk(
+        id=str(doc.id),
+        path=str(doc.field("path") or ""),
+        source=MemorySource(str(doc.field("source") or "")),
+        start_line=int(doc.field("start_line") or 0),
+        end_line=int(doc.field("end_line") or 0),
+        text=str(doc.field("content") or ""),
+        hash=str(doc.field("hash") or ""),
+        embedding=vector,
+    )
+
+
+def _build_source_filter(sources: list[MemorySource] | None) -> str | None:
+    """Build a zvec filter expression for source filtering."""
+    if not sources:
+        return None
+    if len(sources) == 1:
+        return f"source='{_escape(sources[0].value)}'"
+    vals = ", ".join(f"'{_escape(s.value)}'" for s in sources)
+    return f"source IN ({vals})"
+
+
+class ZvecFileStore(BaseFileStore):
+    """Zvec file storage with vector and keyword search.
+
+    Provides zvec-backed persistent storage with:
+    - Vector similarity search (native zvec HNSW)
+    - Keyword search (Python substring matching on fetched results)
+    - Hybrid search (weighted fusion of vector and keyword results)
+
+    Note:
+        Keyword search operates on chunks fetched from zvec, which is subject
+        to the topk limit (1024 in zvec < v0.3.2, 100,000 in v0.3.2+).
+        For collections with more chunks than the topk limit, keyword search
+        may not scan all documents.
+    """
+
+    def __init__(
+        self,
+        store_name: str,
+        db_path: str | Path,
+        embedding_model: Any | None = None,
+        vector_enabled: bool = False,
+        fts_enabled: bool = True,
+        dimension: int = 1024,
+        **kwargs: Any,
+    ):
+        if _ZVEC_IMPORT_ERROR is not None:
+            raise ImportError(
+                "Zvec requires extra dependencies. Install with `pip install zvec`",
+            ) from _ZVEC_IMPORT_ERROR
+
+        super().__init__(
+            store_name=store_name,
+            db_path=db_path,
+            embedding_model=embedding_model,
+            vector_enabled=vector_enabled,
+            fts_enabled=fts_enabled,
+            **kwargs,
+        )
+
+        self.dimension = dimension
+        self._collection = None
+        self._initialized = False
+        self._metadata_file: Path = self.db_path / f"{store_name}_file_metadata.json"
+        self._metadata_cache: dict[str, dict[str, FileMetadata]] = {}
+
+    @property
+    def collection_name(self) -> str:
+        """Get the name of the zvec collection for this store."""
+        return f"chunks_{self.store_name}"
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    async def start(self) -> None:
+        """Initialize zvec engine and open the collection."""
+        if not self._initialized:
+            try:
+                zvec.init()
+            except RuntimeError:
+                pass
+            self._initialized = True
+
+        self.db_path.mkdir(parents=True, exist_ok=True)
+        collection_path = str(self.db_path / self.collection_name)
+        option = CollectionOption(read_only=False, enable_mmap=True)
+
+        try:
+            self._collection = zvec.open(collection_path, option)
+            logger.info(f"Opened existing zvec file store collection: {collection_path}")
+        except Exception:
+            schema = _build_file_store_schema(self.collection_name, self.dimension)
+            self._collection = zvec.create_and_open(
+                path=collection_path,
+                schema=schema,
+                option=option,
+            )
+            logger.info(f"Created new zvec file store collection: {collection_path}")
+
+        self._metadata_cache = await self._load_metadata()
+
+    async def close(self) -> None:
+        """Close zvec collection and persist metadata."""
+        if self._metadata_cache:
+            await self._save_metadata(self._metadata_cache)
+
+        if self._collection is not None:
+            try:
+                self._collection.flush()
+            except Exception as e:
+                logger.warning(f"Failed to flush collection on close: {e}")
+            self._collection = None
+
+    # ------------------------------------------------------------------
+    # Metadata management
+    # ------------------------------------------------------------------
+
+    async def _load_metadata(self) -> dict[str, dict[str, FileMetadata]]:
+        """Load file metadata from JSON file."""
+        if not self._metadata_file.exists():
+            return {}
+        try:
+            data = json.loads(self._metadata_file.read_text(encoding="utf-8"))
+            result: dict[str, dict[str, FileMetadata]] = {}
+            for source, files in data.items():
+                result[source] = {}
+                for path, meta in files.items():
+                    result[source][path] = FileMetadata(**meta)
+            return result
+        except Exception as e:
+            logger.warning(f"Failed to load metadata from {self._metadata_file}: {e}")
+            return {}
+
+    async def _save_metadata(self, metadata: dict[str, dict[str, FileMetadata]]) -> None:
+        """Save file metadata to JSON file."""
+        try:
+            out: dict[str, dict[str, dict]] = {}
+            for source, files in metadata.items():
+                out[source] = {}
+                for path, meta in files.items():
+                    out[source][path] = {
+                        "path": meta.path,
+                        "hash": meta.hash,
+                        "mtime_ms": meta.mtime_ms,
+                        "size": meta.size,
+                        "chunk_count": meta.chunk_count,
+                    }
+            self._metadata_file.write_text(
+                json.dumps(out, indent=2, ensure_ascii=False),
+                encoding="utf-8",
+            )
+        except Exception as e:
+            logger.error(f"Failed to save metadata to {self._metadata_file}: {e}")
+
+    # ------------------------------------------------------------------
+    # CRUD operations
+    # ------------------------------------------------------------------
+
+    async def upsert_file(
+        self,
+        file_meta: FileMetadata,
+        source: MemorySource,
+        chunks: list[MemoryChunk],
+    ) -> None:
+        """Insert or update a file and its chunks."""
+        if not chunks:
+            return
+
+        # Delete existing chunks for this file first
+        await self.delete_file(file_meta.path, source)
+
+        # Generate embeddings
+        chunks = await self.get_chunk_embeddings(chunks)
+
+        file_meta_json = json.dumps(
+            {
+                "path": file_meta.path,
+                "hash": file_meta.hash,
+                "mtime_ms": file_meta.mtime_ms,
+                "size": file_meta.size,
+                "chunk_count": len(chunks),
+            },
+            ensure_ascii=False,
+        )
+
+        docs = [_chunk_to_doc(c, file_meta_json) for c in chunks]
+        self._collection.insert(docs)
+
+        # Update metadata cache
+        if source.value not in self._metadata_cache:
+            self._metadata_cache[source.value] = {}
+        self._metadata_cache[source.value][file_meta.path] = FileMetadata(
+            hash=file_meta.hash,
+            mtime_ms=file_meta.mtime_ms,
+            size=file_meta.size,
+            path=file_meta.path,
+            chunk_count=len(chunks),
+        )
+
+    async def delete_file(self, path: str, source: MemorySource) -> None:
+        """Delete a file and all its chunks."""
+        filter_expr = f"path='{_escape(path)}' AND source='{_escape(source.value)}'"
+        results = self._collection.query(topk=_ZVEC_MAX_TOPK, filter=filter_expr, include_vector=False)
+
+        ids_to_delete = [doc.id for doc in results]
+        if ids_to_delete:
+            self._collection.delete(ids_to_delete)
+
+        if source.value in self._metadata_cache:
+            self._metadata_cache[source.value].pop(path, None)
+
+    async def delete_file_chunks(self, path: str, chunk_ids: list[str]) -> None:
+        """Delete specific chunks for a file."""
+        if not chunk_ids:
+            return
+        self._collection.delete(chunk_ids)
+
+    async def upsert_chunks(
+        self,
+        chunks: list[MemoryChunk],
+        source: MemorySource,
+    ) -> None:
+        """Insert or update specific chunks."""
+        if not chunks:
+            return
+
+        chunks = await self.get_chunk_embeddings(chunks)
+        docs = [_chunk_to_doc(c) for c in chunks]
+        self._collection.upsert(docs)
+
+    # ------------------------------------------------------------------
+    # Listing and metadata
+    # ------------------------------------------------------------------
+
+    async def list_files(self, source: MemorySource) -> list[str]:
+        """List all indexed files for a source."""
+        if source.value not in self._metadata_cache:
+            return []
+        return list(self._metadata_cache[source.value].keys())
+
+    async def get_file_metadata(
+        self,
+        path: str,
+        source: MemorySource,
+    ) -> FileMetadata | None:
+        """Get file metadata."""
+        if source.value not in self._metadata_cache:
+            return None
+        return self._metadata_cache[source.value].get(path)
+
+    async def update_file_metadata(self, file_meta: FileMetadata, source: MemorySource) -> None:
+        """Update file metadata without affecting chunks."""
+        if source.value not in self._metadata_cache:
+            self._metadata_cache[source.value] = {}
+        self._metadata_cache[source.value][file_meta.path] = FileMetadata(
+            hash=file_meta.hash,
+            mtime_ms=file_meta.mtime_ms,
+            size=file_meta.size,
+            path=file_meta.path,
+            chunk_count=file_meta.chunk_count,
+        )
+
+    async def get_file_chunks(
+        self,
+        path: str,
+        source: MemorySource,
+    ) -> list[MemoryChunk]:
+        """Get all chunks for a file."""
+        filter_expr = f"path='{_escape(path)}' AND source='{_escape(source.value)}'"
+        results = self._collection.query(
+            topk=_ZVEC_MAX_TOPK,
+            filter=filter_expr,
+            include_vector=True,
+        )
+        chunks = [_doc_to_chunk(doc) for doc in results]
+        chunks.sort(key=lambda c: c.start_line)
+        return chunks
+
+    # ------------------------------------------------------------------
+    # Search
+    # ------------------------------------------------------------------
+
+    async def vector_search(
+        self,
+        query: str,
+        limit: int,
+        sources: list[MemorySource] | None = None,
+    ) -> list[MemorySearchResult]:
+        """Perform vector similarity search."""
+        if not self.vector_enabled or not query:
+            return []
+
+        query_embedding = await self.get_embedding(query)
+        if not query_embedding:
+            return []
+
+        filter_expr = _build_source_filter(sources)
+        vq = VectorQuery(field_name=_DEFAULT_VECTOR_FIELD, vector=query_embedding)
+
+        try:
+            results = self._collection.query(
+                vectors=vq,
+                topk=min(limit, _ZVEC_MAX_TOPK),
+                filter=filter_expr,
+                include_vector=False,
+            )
+        except Exception as e:
+            logger.error(f"Vector search failed: {e}")
+            return []
+
+        search_results = []
+        for doc in results:
+            score = doc.score if doc.score is not None else 0.0
+            # zvec cosine score might need normalization depending on version
+            search_results.append(
+                MemorySearchResult(
+                    path=str(doc.field("path") or ""),
+                    start_line=int(doc.field("start_line") or 0),
+                    end_line=int(doc.field("end_line") or 0),
+                    score=score,
+                    snippet=str(doc.field("content") or ""),
+                    source=MemorySource(str(doc.field("source") or "")),
+                    raw_metric=score,
+                ),
+            )
+
+        search_results.sort(key=lambda r: r.score, reverse=True)
+        return search_results[:limit]
+
+    async def keyword_search(
+        self,
+        query: str,
+        limit: int,
+        sources: list[MemorySource] | None = None,
+    ) -> list[MemorySearchResult]:
+        """Perform keyword search via Python substring matching.
+
+        Fetches chunks from zvec (subject to topk limit) then matches
+        keywords in Python. For collections larger than the topk limit,
+        not all documents are scanned.
+        """
+        if not self.fts_enabled or not query:
+            return []
+
+        words = query.split()
+        if not words:
+            return []
+
+        # Fetch candidate chunks from zvec
+        filter_expr = _build_source_filter(sources)
+        results = self._collection.query(
+            topk=_ZVEC_MAX_TOPK,
+            filter=filter_expr,
+            include_vector=False,
+        )
+
+        query_lower = query.lower()
+        words_lower = [w.lower() for w in words]
+        n_words = len(words)
+
+        search_results = []
+        for doc in results:
+            text = str(doc.field("content") or "")
+            text_lower = text.lower()
+            match_count = sum(1 for w in words_lower if w in text_lower)
+            if match_count == 0:
+                continue
+
+            base_score = match_count / n_words
+            phrase_bonus = 0.2 if n_words > 1 and query_lower in text_lower else 0.0
+            score = min(1.0, base_score + phrase_bonus)
+
+            search_results.append(
+                MemorySearchResult(
+                    path=str(doc.field("path") or ""),
+                    start_line=int(doc.field("start_line") or 0),
+                    end_line=int(doc.field("end_line") or 0),
+                    score=score,
+                    snippet=text,
+                    source=MemorySource(str(doc.field("source") or "")),
+                ),
+            )
+
+        search_results.sort(key=lambda r: r.score, reverse=True)
+        return search_results[:limit]
+
+    async def hybrid_search(
+        self,
+        query: str,
+        limit: int,
+        sources: list[MemorySource] | None = None,
+        vector_weight: float = 0.7,
+        candidate_multiplier: float = 3.0,
+    ) -> list[MemorySearchResult]:
+        """Perform hybrid search combining vector and keyword search."""
+        assert 0.0 <= vector_weight <= 1.0, f"vector_weight must be between 0 and 1, got {vector_weight}"
+
+        candidates = min(200, max(1, int(limit * candidate_multiplier)))
+        text_weight = 1.0 - vector_weight
+
+        if self.vector_enabled and self.fts_enabled:
+            keyword_results = await self.keyword_search(query, candidates, sources)
+            vector_results = await self.vector_search(query, candidates, sources)
+
+            if not keyword_results:
+                return vector_results[:limit]
+            elif not vector_results:
+                return keyword_results[:limit]
+            else:
+                return self._merge_hybrid_results(
+                    vector=vector_results,
+                    keyword=keyword_results,
+                    vector_weight=vector_weight,
+                    text_weight=text_weight,
+                )[:limit]
+        elif self.vector_enabled:
+            return await self.vector_search(query, limit, sources)
+        elif self.fts_enabled:
+            return await self.keyword_search(query, limit, sources)
+        else:
+            return []
+
+    @staticmethod
+    def _merge_hybrid_results(
+        vector: list[MemorySearchResult],
+        keyword: list[MemorySearchResult],
+        vector_weight: float,
+        text_weight: float,
+    ) -> list[MemorySearchResult]:
+        """Merge vector and keyword search results with weighted scoring."""
+        merged: dict[str, MemorySearchResult] = {}
+
+        for result in vector:
+            result.score = result.score * vector_weight
+            merged[result.merge_key] = result
+
+        for result in keyword:
+            key = result.merge_key
+            if key in merged:
+                merged[key].score += result.score * text_weight
+            else:
+                result.score = result.score * text_weight
+                merged[key] = result
+
+        results = list(merged.values())
+        results.sort(key=lambda r: r.score, reverse=True)
+        return results
+
+    # ------------------------------------------------------------------
+    # Maintenance
+    # ------------------------------------------------------------------
+
+    async def clear_all(self) -> None:
+        """Clear all indexed data."""
+        # Delete all documents
+        stats = self._collection.stats
+        count = stats.doc_count if stats else 0
+        if count > 0:
+            try:
+                self._collection.delete_by_filter("content!=''")
+            except Exception:
+                remaining = count
+                while remaining > 0:
+                    batch = self._collection.query(
+                        topk=min(remaining, _ZVEC_MAX_TOPK),
+                        include_vector=False,
+                    )
+                    if not batch:
+                        break
+                    self._collection.delete([doc.id for doc in batch])
+                    remaining -= len(batch)
+
+        self._metadata_cache = {}
+        await self._save_metadata({})
+        logger.info(f"Cleared all data from zvec file store: {self.collection_name}")

reme/core/file_watcher/base_file_watcher.py

@@ -76,12 +76,14 @@ class BaseFileWatcher:
         if self._running:
             return
 
+        self._stop_event = asyncio.Event()
         self._running = True
 
         async def _initialize_and_watch():
             if self.rebuild_index_on_start:
-                await self.file_store.clear_all()
-                logger.info("Cleared all indexed data on start")
+                if self.file_store is not None:
+                    await self.file_store.clear_all()
+                    logger.info("Cleared all indexed data on start")
                 await self._scan_existing_files()
             await self._watch_loop()
 
@@ -183,6 +185,7 @@ class BaseFileWatcher:
                 logger.info(f"Starting watch on valid paths: {valid_paths}")
                 async for changes in awatch(
                     *valid_paths,
+                    force_polling=True,
                     watch_filter=self.watch_filter,
                     recursive=self.recursive,
                     debounce=self.debounce,

reme/core/schema/service_config.py

@@ -122,6 +122,7 @@ class ServiceConfig(BasicConfig):
     )
     ray_max_workers: int = Field(default=-1)
     log_to_console: bool = Field(default=True)
+    log_to_file: bool = Field(default=True)
     disabled_flows: list[str] = Field(default_factory=list)
     enabled_flows: list[str] = Field(default_factory=list)
 

reme/core/service_context.py

@@ -34,6 +34,7 @@ class ServiceContext(BaseDict):
         config_path: str | None = None,
         enable_logo: bool = True,
         log_to_console: bool = True,
+        log_to_file: bool = True,
         default_as_llm_config: dict | None = None,
         default_as_llm_formatter_config: dict | None = None,
         default_as_token_counter_config: dict | None = None,
@@ -79,6 +80,7 @@ class ServiceContext(BaseDict):
                 {
                     "enable_logo": enable_logo,
                     "log_to_console": log_to_console,
+                    "log_to_file": log_to_file,
                     "working_dir": working_dir,
                 },
             )

reme/core/utils/logger_utils.py

@@ -5,13 +5,19 @@ import sys
 from datetime import datetime
 
 
-def init_logger(log_dir: str = "logs", level: str = "INFO", log_to_console: bool = True) -> None:
+def init_logger(
+    log_dir: str = "logs",
+    level: str = "INFO",
+    log_to_console: bool = True,
+    log_to_file: bool = True,
+) -> None:
     """Initialize the logger with both file and console handlers.
 
     Args:
         log_dir: Directory path for log files
         level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
         log_to_console: Whether to print logs to console/screen
+        log_to_file: Whether to persist logs to files under log_dir
     """
     from loguru import logger
 
@@ -28,25 +34,26 @@ def init_logger(log_dir: str = "logs", level: str = "INFO", log_to_console: bool
         )
 
     # Try to configure file-based logging (skip if permission denied)
-    try:
-        # Ensure the logging directory exists
-        os.makedirs(log_dir, exist_ok=True)
+    if log_to_file:
+        try:
+            # Ensure the logging directory exists
+            os.makedirs(log_dir, exist_ok=True)
 
-        # Generate filename based on the current timestamp
-        # Use dashes instead of colons for Windows compatibility
-        current_ts = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
-        log_filename = f"{current_ts}.log"
-        log_filepath = os.path.join(log_dir, log_filename)
+            # Generate filename based on the current timestamp
+            # Use dashes instead of colons for Windows compatibility
+            current_ts = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+            log_filename = f"{current_ts}.log"
+            log_filepath = os.path.join(log_dir, log_filename)
 
-        # Configure file-based logging with rotation and compression
-        logger.add(
-            log_filepath,
-            level=level,
-            rotation="00:00",
-            retention="7 days",
-            compression="zip",
-            encoding="utf-8",
-            format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {file}:{line} | {function} | {message}",
-        )
-    except Exception as e:
-        logger.error(f"Error configuring file logging: {e}")
+            # Configure file-based logging with rotation and compression
+            logger.add(
+                log_filepath,
+                level=level,
+                rotation="00:00",
+                retention="7 days",
+                compression="zip",
+                encoding="utf-8",
+                format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {file}:{line} | {function} | {message}",
+            )
+        except Exception as e:
+            logger.error(f"Error configuring file logging: {e}")

reme/core/vector_store/__init__.py

@@ -3,28 +3,34 @@
 from .base_vector_store import BaseVectorStore
 from .chroma_vector_store import ChromaVectorStore
 from .es_vector_store import ESVectorStore
+from .hologres_store import HologresVectorStore
 from .local_vector_store import LocalVectorStore
 from .obvec_vector_store import ObVecVectorStore
 from .pgvector_store import PGVectorStore
 from .qdrant_vector_store import QdrantVectorStore
+from .zvec_vector_store import ZvecVectorStore
 from ..registry_factory import R
 
 __all__ = [
     "BaseVectorStore",
     "ChromaVectorStore",
     "ESVectorStore",
+    "HologresVectorStore",
     "LocalVectorStore",
     "ObVecVectorStore",
     "PGVectorStore",
     "QdrantVectorStore",
+    "ZvecVectorStore",
 ]
 
 R.vector_stores.register("chroma")(ChromaVectorStore)
 R.vector_stores.register("es")(ESVectorStore)
+R.vector_stores.register("hologres")(HologresVectorStore)
 R.vector_stores.register("local")(LocalVectorStore)
 R.vector_stores.register("obvec")(ObVecVectorStore)
 R.vector_stores.register("pgvector")(PGVectorStore)
 R.vector_stores.register("qdrant")(QdrantVectorStore)
+R.vector_stores.register("zvec")(ZvecVectorStore)
 
 try:
     from .seekdb_vector_store import SeekdbVectorStore

reme/core/vector_store/hologres_store.py

@@ -0,0 +1,633 @@
+"""Hologres implementation for vector storage and retrieval."""
+
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+
+from .base_vector_store import BaseVectorStore
+from ..embedding import BaseEmbeddingModel
+from ..schema import VectorNode
+
+_ASYNCPG_IMPORT_ERROR: Exception | None = None
+
+try:
+    import asyncpg
+    from asyncpg import Pool
+except Exception as e:
+    _ASYNCPG_IMPORT_ERROR = e
+    asyncpg = None
+    Pool = None
+
+
+class HologresVectorStore(BaseVectorStore):
+    """Vector store implementation using Hologres for efficient similarity search.
+
+    Hologres uses native float4[] arrays for vector storage with built-in
+    HGraph index for approximate nearest neighbor search, unlike pgvector
+    which requires an extension.
+    """
+
+    @staticmethod
+    def _validate_table_name(name: str) -> None:
+        """Validate table name to prevent SQL injection."""
+        if not name:
+            raise ValueError("Table name cannot be empty")
+        if len(name) > 63:
+            raise ValueError(f"Table name too long: {len(name)} characters (max 63)")
+        if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", name):
+            raise ValueError(
+                f"Invalid table name: {name}. Must start with letter or underscore, "
+                "and contain only alphanumeric characters and underscores.",
+            )
+
+    def __init__(
+        self,
+        collection_name: str,
+        db_path: str | Path,
+        embedding_model: BaseEmbeddingModel,
+        host: str = "localhost",
+        port: int = 80,
+        database: str = "postgres",
+        user: str = "postgres",
+        password: str = "",
+        schema: str = "public",
+        min_size: int = 1,
+        max_size: int = 10,
+        dsn: str | None = None,
+        distance_method: str = "Cosine",
+        **kwargs,
+    ):
+        """Initialize the Hologres vector store with connection parameters.
+
+        Args:
+            collection_name: Name of the collection (table).
+            db_path: Database path (used by base class).
+            embedding_model: Embedding model for generating vectors.
+            host: Hologres host address.
+            port: Hologres port (default 80 for Hologres).
+            database: Database name.
+            user: Database user.
+            password: Database password.
+            schema: PostgreSQL schema name (default "public").
+            min_size: Minimum connections in pool.
+            max_size: Maximum connections in pool.
+            dsn: Full DSN connection string (overrides individual params).
+            distance_method: Distance method for HGraph index (Cosine, InnerProduct, Euclidean).
+        """
+        if _ASYNCPG_IMPORT_ERROR is not None:
+            raise ImportError(
+                "Hologres vector store requires asyncpg. Install with `pip install asyncpg`",
+            ) from _ASYNCPG_IMPORT_ERROR
+
+        self._validate_table_name(collection_name)
+        self._validate_table_name(schema)
+
+        super().__init__(
+            collection_name=collection_name,
+            db_path=db_path,
+            embedding_model=embedding_model,
+            **kwargs,
+        )
+
+        self.dsn = dsn
+        self.host = host
+        self.port = port
+        self.database = database
+        self.user = user
+        self.password = password
+        self.schema = schema
+        self.min_size = min_size
+        self.max_size = max_size
+        self.distance_method = distance_method
+        self._pool: Pool | None = None
+        self.embedding_model_dims = embedding_model.dimensions
+
+    @property
+    def _qualified_name(self) -> str:
+        """Return the schema-qualified table name (e.g. 'my_schema.my_table')."""
+        return f"{self.schema}.{self.collection_name}"
+
+    def _qualify(self, table_name: str) -> str:
+        """Return a schema-qualified name for an arbitrary table."""
+        return f"{self.schema}.{table_name}"
+
+    @staticmethod
+    async def _hologres_reset(conn):
+        """Custom reset for Hologres connections."""
+        await conn.execute(
+            """
+            SELECT pg_advisory_unlock_all();
+            CLOSE ALL;
+            RESET ALL;
+        """,
+        )
+
+    async def _get_pool(self) -> Pool:
+        """Create or return the existing asyncpg connection pool."""
+        if self._pool is None:
+            if self.dsn:
+                self._pool = await asyncpg.create_pool(
+                    dsn=self.dsn,
+                    min_size=self.min_size,
+                    max_size=self.max_size,
+                    reset=self._hologres_reset,
+                )
+            else:
+                self._pool = await asyncpg.create_pool(
+                    host=self.host,
+                    port=self.port,
+                    database=self.database,
+                    user=self.user,
+                    password=self.password,
+                    min_size=self.min_size,
+                    max_size=self.max_size,
+                    reset=self._hologres_reset,
+                )
+
+            # Ensure schema exists
+            async with self._pool.acquire() as conn:
+                await conn.execute(f"CREATE SCHEMA IF NOT EXISTS {self.schema}")
+
+            logger.info(f"Hologres connection pool created for database {self.database}")
+
+        return self._pool
+
+    @staticmethod
+    def _vector_to_pg_array(vector: list[float]) -> str:
+        """Convert a Python list of floats to PostgreSQL array literal format."""
+        return "{" + ",".join(map(str, vector)) + "}"
+
+    @staticmethod
+    def _pg_array_to_vector(pg_array) -> list[float] | None:
+        """Convert a PostgreSQL array result to a Python list of floats."""
+        if pg_array is None:
+            return None
+        if isinstance(pg_array, list):
+            return [float(x) for x in pg_array]
+        # Handle string format like {1.0,2.0,3.0}
+        raw = str(pg_array)
+        if raw.startswith("{") and raw.endswith("}"):
+            return [float(x) for x in raw[1:-1].split(",")]
+        return None
+
+    async def list_collections(self) -> list[str]:
+        """List all available table names in the current schema."""
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            rows = await conn.fetch(
+                "SELECT table_name FROM information_schema.tables WHERE table_schema = $1",
+                self.schema,
+            )
+            return [row["table_name"] for row in rows]
+
+    async def create_collection(self, collection_name: str, **kwargs):
+        """Create a new Hologres table with vector support and HGraph index."""
+        self._validate_table_name(collection_name)
+        pool = await self._get_pool()
+        dimensions = kwargs.get("dimensions", self.embedding_model_dims)
+        qualified = self._qualify(collection_name)
+
+        async with pool.acquire() as conn:
+            create_sql = f"""
+                CREATE TABLE IF NOT EXISTS {qualified} (
+                    id TEXT PRIMARY KEY,
+                    content TEXT,
+                    vector float4[] CHECK (array_ndims(vector) = 1 AND array_length(vector, 1) = {dimensions}),
+                    metadata JSONB
+                )
+                WITH (
+                    vectors = '{{
+                        "vector": {{
+                            "algorithm": "HGraph",
+                            "distance_method": "{self.distance_method}",
+                            "builder_params": {{
+                                "base_quantization_type": "rabitq",
+                                "rabitq_use_fht":true,
+                                "graph_storage_type": "compressed",
+                                "max_total_size_to_merge_mb": 4096,
+                                "max_degree": 64,
+                                "ef_construction": 400,
+                                "precise_quantization_type": "fp32",
+                                "use_reorder": true
+                            }}
+                        }}
+                    }}'
+                )
+            """
+            await conn.execute(create_sql)
+
+        logger.info(f"Created Hologres collection {qualified} with dimensions={dimensions}")
+
+    async def delete_collection(self, collection_name: str, **kwargs):
+        """Remove the specified collection table from the database."""
+        self._validate_table_name(collection_name)
+        pool = await self._get_pool()
+        qualified = self._qualify(collection_name)
+        async with pool.acquire() as conn:
+            await conn.execute(f"DROP TABLE IF EXISTS {qualified}")
+        logger.info(f"Deleted collection {qualified}")
+
+    async def copy_collection(self, collection_name: str, **kwargs):
+        """Duplicate the structure and content of the current collection to a new table."""
+        self._validate_table_name(collection_name)
+        pool = await self._get_pool()
+        qualified_src = self._qualified_name
+        qualified_dst = self._qualify(collection_name)
+
+        async with pool.acquire() as conn:
+            columns = await conn.fetch(
+                """
+                SELECT column_name, data_type, udt_name
+                FROM information_schema.columns
+                WHERE table_name = $1 AND table_schema = $2
+            """,
+                self.collection_name,
+                self.schema,
+            )
+
+            if not columns:
+                raise ValueError(f"Source collection {qualified_src} does not exist")
+
+            # Create new table with primary key, then add data
+            await conn.execute(
+                f"""
+                               SET hg_experimental_enable_create_table_like_properties = true;
+                               CALL hg_create_table_like('{qualified_dst}', 'select * from {qualified_src}')
+                               """,
+            )
+            await conn.execute(f"INSERT INTO {qualified_dst} SELECT * FROM {qualified_src} ;")
+
+        logger.info(f"Copied collection {qualified_src} to {qualified_dst}")
+
+    async def insert(self, nodes: VectorNode | list[VectorNode], **kwargs):
+        """Insert or upsert vector nodes into the Hologres collection."""
+        if isinstance(nodes, VectorNode):
+            nodes = [nodes]
+
+        if not nodes:
+            return
+
+        nodes_without_vectors = [node for node in nodes if node.vector is None]
+        if nodes_without_vectors:
+            nodes_with_vectors = await self.get_node_embeddings(nodes_without_vectors)
+            vector_map = {n.vector_id: n for n in nodes_with_vectors}
+            nodes_to_insert = [vector_map.get(n.vector_id, n) if n.vector is None else n for n in nodes]
+        else:
+            nodes_to_insert = nodes
+
+        pool = await self._get_pool()
+        data = [
+            (
+                node.vector_id,
+                node.content,
+                node.vector,
+                json.dumps(node.metadata),
+            )
+            for node in nodes_to_insert
+        ]
+
+        async with pool.acquire() as conn:
+            on_conflict = kwargs.get("on_conflict", "update")
+
+            if on_conflict == "update":
+                await conn.executemany(
+                    f"""
+                    INSERT INTO {self._qualified_name} (id, content, vector, metadata)
+                    VALUES ($1, $2, $3::float4[], $4::jsonb)
+                    ON CONFLICT (id) DO UPDATE SET
+                        content = EXCLUDED.content,
+                        vector = EXCLUDED.vector,
+                        metadata = EXCLUDED.metadata
+                """,
+                    data,
+                )
+            elif on_conflict == "ignore":
+                await conn.executemany(
+                    f"""
+                    INSERT INTO {self._qualified_name} (id, content, vector, metadata)
+                    VALUES ($1, $2, $3::float4[], $4::jsonb)
+                    ON CONFLICT (id) DO NOTHING
+                """,
+                    data,
+                )
+            else:
+                await conn.executemany(
+                    f"""
+                    INSERT INTO {self._qualified_name} (id, content, vector, metadata)
+                    VALUES ($1, $2, $3::float4[], $4::jsonb)
+                """,
+                    data,
+                )
+
+        logger.info(f"Inserted {len(nodes_to_insert)} documents into {self._qualified_name}")
+
+    @staticmethod
+    def _build_filter_clause(filters: dict | None) -> tuple[str, list]:
+        """Generate an SQL WHERE clause and parameter list from a filter dictionary.
+
+        Supports two filter formats:
+        1. Range query: {"field": [start_value, end_value]}
+        2. Exact match: {"field": value}
+        """
+        if not filters:
+            return "", []
+
+        conditions = []
+        params = []
+        param_idx = 1
+
+        for key, value in filters.items():
+            if not key.replace("_", "").replace(".", "").isalnum():
+                raise ValueError(
+                    f"Invalid metadata key: {key}. Only alphanumeric characters, underscore and dot are allowed.",
+                )
+
+            if isinstance(value, list) and len(value) == 2:
+                if isinstance(value[0], (int, float)) and isinstance(value[1], (int, float)):
+                    conditions.append(
+                        f"(metadata->>'{key}')::numeric >= ${param_idx} AND "
+                        f"(metadata->>'{key}')::numeric <= ${param_idx + 1}",
+                    )
+                else:
+                    conditions.append(f"metadata->>'{key}' >= ${param_idx} AND metadata->>'{key}' <= ${param_idx + 1}")
+                params.extend([value[0], value[1]])
+                param_idx += 2
+            else:
+                conditions.append(f"metadata->>'{key}' = ${param_idx}")
+                params.append(str(value))
+                param_idx += 1
+
+        filter_clause = "WHERE " + " AND ".join(conditions) if conditions else ""
+        return filter_clause, params
+
+    async def search(
+        self,
+        query: str,
+        limit: int = 5,
+        filters: dict | None = None,
+        **kwargs,
+    ) -> list[VectorNode]:
+        """Perform vector similarity search using Hologres approx_cosine_distance."""
+        query_vector = await self.get_embedding(query)
+        vector_str = self._vector_to_pg_array(query_vector)
+        pool = await self._get_pool()
+
+        filter_clause, filter_params = self._build_filter_clause(filters)
+
+        # filter_params use $1..$N, limit uses $(N+1)
+        limit_placeholder = f"${len(filter_params) + 1}"
+
+        async with pool.acquire() as conn:
+            sql = f"""
+                SELECT id, content, vector, metadata,
+                       approx_cosine_distance(vector, '{vector_str}') AS distance
+                FROM {self._qualified_name}
+                {filter_clause}
+                ORDER BY distance DESC
+                LIMIT {limit_placeholder}
+            """
+            rows = await conn.fetch(sql, *filter_params, limit)
+
+        results = []
+        score_threshold = kwargs.get("score_threshold")
+
+        for row in rows:
+            distance = float(row["distance"])
+            # approx_cosine_distance returns cosine similarity (higher = more similar)
+            score = distance
+            if score_threshold is not None and score < score_threshold:
+                continue
+
+            vector_data = self._pg_array_to_vector(row["vector"])
+
+            metadata = row["metadata"] if row["metadata"] else {}
+            if isinstance(metadata, str):
+                metadata = json.loads(metadata)
+
+            metadata["score"] = score
+            metadata["_distance"] = 1 - score
+
+            node = VectorNode(
+                vector_id=row["id"],
+                content=row["content"] or "",
+                vector=vector_data,
+                metadata=metadata,
+            )
+            results.append(node)
+
+        return results
+
+    async def delete(self, vector_ids: str | list[str], **kwargs):
+        """Remove specific vector records from the collection by their IDs."""
+        if isinstance(vector_ids, str):
+            vector_ids = [vector_ids]
+
+        if not vector_ids:
+            return
+
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            placeholders = ", ".join([f"${i + 1}" for i in range(len(vector_ids))])
+            await conn.execute(
+                f"DELETE FROM {self._qualified_name} WHERE id IN ({placeholders})",
+                *vector_ids,
+            )
+
+        logger.info(f"Deleted {len(vector_ids)} documents from {self._qualified_name}")
+
+    async def delete_all(self, **kwargs):
+        """Remove all vectors from the collection."""
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            result = await conn.execute(f"DELETE FROM {self._qualified_name}")
+
+        logger.info(f"Deleted all documents from {self._qualified_name} result={result}")
+
+    async def update(self, nodes: VectorNode | list[VectorNode], **kwargs):
+        """Update existing vector nodes with new content, embeddings, or metadata."""
+        if isinstance(nodes, VectorNode):
+            nodes = [nodes]
+
+        if not nodes:
+            return
+
+        nodes_without_vectors = [node for node in nodes if node.vector is None and node.content]
+        if nodes_without_vectors:
+            nodes_with_vectors = await self.get_node_embeddings(nodes_without_vectors)
+            vector_map = {n.vector_id: n for n in nodes_with_vectors}
+            nodes_to_update = [vector_map.get(n.vector_id, n) if n.vector is None and n.content else n for n in nodes]
+        else:
+            nodes_to_update = nodes
+
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            for node in nodes_to_update:
+                update_fields = []
+                params = []
+                idx = 1
+
+                if node.content:
+                    update_fields.append(f"content = ${idx}")
+                    params.append(node.content)
+                    idx += 1
+
+                if node.vector:
+                    update_fields.append(f"vector = ${idx}::float4[]")
+                    params.append(node.vector)
+                    idx += 1
+
+                if node.metadata:
+                    update_fields.append(f"metadata = ${idx}::jsonb")
+                    params.append(json.dumps(node.metadata))
+                    idx += 1
+
+                if update_fields:
+                    params.append(node.vector_id)
+                    await conn.execute(
+                        f"UPDATE {self._qualified_name} SET {', '.join(update_fields)} WHERE id = ${idx}",
+                        *params,
+                    )
+
+        logger.info(f"Updated {len(nodes_to_update)} documents in {self._qualified_name}")
+
+    async def get(self, vector_ids: str | list[str]) -> VectorNode | list[VectorNode] | None:
+        """Retrieve vector nodes by their unique identifiers."""
+        single_result = isinstance(vector_ids, str)
+        if single_result:
+            vector_ids = [vector_ids]
+
+        if not vector_ids:
+            return [] if not single_result else None
+
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            placeholders = ", ".join([f"${i + 1}" for i in range(len(vector_ids))])
+            rows = await conn.fetch(
+                f"SELECT id, content, vector, metadata FROM {self._qualified_name} WHERE id IN ({placeholders})",
+                *vector_ids,
+            )
+
+        results = []
+        for row in rows:
+            vector_data = self._pg_array_to_vector(row["vector"])
+
+            metadata = row["metadata"] if row["metadata"] else {}
+            if isinstance(metadata, str):
+                metadata = json.loads(metadata)
+
+            results.append(
+                VectorNode(
+                    vector_id=row["id"],
+                    content=row["content"] or "",
+                    vector=vector_data,
+                    metadata=metadata,
+                ),
+            )
+
+        if single_result:
+            return results[0] if results else None
+        return results
+
+    async def list(
+        self,
+        filters: dict | None = None,
+        limit: int | None = None,
+        sort_key: str | None = None,
+        reverse: bool = False,
+    ) -> list[VectorNode]:
+        """Return a list of vector nodes matching the provided filters and limit.
+
+        Args:
+            filters: Dictionary of filter conditions to match vectors
+            limit: Maximum number of vectors to return
+            sort_key: Key to sort the results by (e.g., field name in metadata). None for no sorting
+            reverse: If True, sort in descending order; if False, sort in ascending order
+        """
+        pool = await self._get_pool()
+        filter_clause, filter_params = self._build_filter_clause(filters)
+
+        order_clause = ""
+        if sort_key:
+            order_direction = "DESC" if reverse else "ASC"
+            order_clause = f"ORDER BY metadata->>'{sort_key}' {order_direction}"
+
+        limit_clause = ""
+        if limit:
+            limit_clause = f"LIMIT ${len(filter_params) + 1}"
+            filter_params.append(limit)
+
+        async with pool.acquire() as conn:
+            sql = f"""
+                SELECT id, content, vector, metadata
+                FROM {self._qualified_name}
+                {filter_clause}
+                {order_clause}
+                {limit_clause}
+            """
+            rows = await conn.fetch(sql, *filter_params)
+
+        results = []
+        for row in rows:
+            vector_data = self._pg_array_to_vector(row["vector"])
+
+            metadata = row["metadata"] if row["metadata"] else {}
+            if isinstance(metadata, str):
+                metadata = json.loads(metadata)
+
+            results.append(
+                VectorNode(
+                    vector_id=row["id"],
+                    content=row["content"] or "",
+                    vector=vector_data,
+                    metadata=metadata,
+                ),
+            )
+
+        return results
+
+    async def collection_info(self) -> dict[str, Any]:
+        """Fetch metadata including record count and disk usage for the collection."""
+        pool = await self._get_pool()
+        qualified = self._qualified_name
+
+        async with pool.acquire() as conn:
+            count = await conn.fetchval(f"SELECT COUNT(*) FROM {qualified}")
+            size = await conn.fetchval(f"SELECT pg_size_pretty(pg_total_relation_size('{qualified}'))")
+
+        return {
+            "name": qualified,
+            "count": count,
+            "size": size,
+        }
+
+    async def reset(self):
+        """Purge all data by dropping and recreating the collection table."""
+        logger.warning(f"Resetting collection {self._qualified_name}...")
+        await self.delete_collection(self.collection_name)
+        await self.create_collection(self.collection_name)
+
+    async def reset_collection(self, collection_name: str):
+        """Reset collection with table name validation."""
+        self._validate_table_name(collection_name)
+        self.collection_name = collection_name
+        await self.create_collection(collection_name)
+        logger.info(f"Collection reset to {self._qualified_name}")
+
+    async def start(self) -> None:
+        """Initialize the PGVector store.
+
+        Creates the connection pool and ensures the collection table exists.
+        """
+        await self._get_pool()
+        await super().start()
+        logger.info(f"Hologres collection {self._qualified_name} initialized")
+
+    async def close(self):
+        """Terminate the database connection pool."""
+        if self._pool is not None:
+            await self._pool.close()
+            self._pool = None
+            logger.info("Hologres connection pool closed")

reme/core/vector_store/zvec_vector_store.py

@@ -0,0 +1,809 @@
+"""Zvec vector store implementation for the ReMe framework."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+
+from .base_vector_store import BaseVectorStore
+from ..embedding import BaseEmbeddingModel
+from ..schema import VectorNode
+
+_ZVEC_IMPORT_ERROR: Exception | None = None
+
+try:
+    import zvec  # type: ignore[import-untyped]
+    from zvec import (
+        CollectionOption,
+        CollectionSchema,
+        DataType,
+        Doc,
+        FieldSchema,
+        HnswIndexParam,
+        InvertIndexParam,
+        VectorQuery,
+        VectorSchema,
+    )
+    from zvec.typing import MetricType
+except Exception as e:
+    _ZVEC_IMPORT_ERROR = e
+    zvec = None  # type: ignore[assignment]
+
+
+# Default vector field name used inside zvec collections
+_DEFAULT_VECTOR_FIELD = "embedding"
+
+# Default scalar content field name for storing text
+_CONTENT_FIELD = "content"
+
+# Field name for JSON-serialized metadata
+_METADATA_FIELD = "metadata"
+
+# Metadata fields promoted to top-level zvec schema columns for native filtering.
+# These are the most commonly filtered keys in ReMe's memory system.
+# Defining them as independent schema columns allows zvec to perform
+# filtering at the database level instead of Python post-filtering.
+# Format: {metadata_key: (zvec_data_type_str, has_inverted_index)}
+_PROMOTED_FIELD_SPECS: dict[str, tuple[str, bool]] = {
+    "memory_type": ("STRING", True),  # Inverted index for exact match filtering
+    "memory_target": ("STRING", True),  # Inverted index for exact match filtering
+    "author": ("STRING", False),
+    "time_int": ("INT64", False),  # Numeric for range queries
+}
+
+# zvec data-type string → DataType enum mapping (populated after import)
+_DATATYPE_MAP: dict[str, Any] = {}  # filled in _build_collection_schema
+
+
+def _escape_zvec_string(value: str) -> str:
+    """Escape a string value for use in zvec filter expressions."""
+    return value.replace("'", "\\'")
+
+
+def _build_zvec_filter(
+    filters: dict | None,
+    promoted_fields: set[str],
+) -> tuple[str | None, dict | None]:
+    """Split ReMe filter dict into a zvec native filter expression and remaining post-filters.
+
+    For filter keys that correspond to promoted schema fields, native
+    zvec filter expressions are generated.  Non-promoted keys are
+    kept for Python post-filtering.
+
+    Args:
+        filters: ReMe-style filter dictionary.
+        promoted_fields: Set of metadata keys that exist as top-level schema columns.
+
+    Returns:
+        (native_filter_expr, post_filter_dict) — either may be None.
+    """
+    if not filters:
+        return None, None
+
+    native_conditions: list[str] = []
+    post_filters: dict = {}
+
+    for key, value in filters.items():
+        if key.startswith("$"):
+            # Compound operators ($or, $and, $not) — keep for post-filtering
+            post_filters[key] = value
+            continue
+
+        if key not in promoted_fields:
+            # Not a promoted field — use post-filtering
+            post_filters[key] = value
+            continue
+
+        # Build native filter condition for promoted fields
+        field_type = _PROMOTED_FIELD_SPECS.get(key, ("STRING", False))[0]
+
+        if isinstance(value, list) and len(value) == 2:
+            # Range query: [start, end]
+            if field_type == "INT64":
+                native_conditions.append(f"{key} >= {value[0]} AND {key} <= {value[1]}")
+            else:
+                # STRING range — use >= and <= with string escaping
+                native_conditions.append(
+                    f"{key} >= '{_escape_zvec_string(str(value[0]))}' "
+                    f"AND {key} <= '{_escape_zvec_string(str(value[1]))}'",
+                )
+        elif isinstance(value, bool):
+            native_conditions.append(f"{key} = {str(value).upper()}")
+        elif isinstance(value, (int, float)):
+            native_conditions.append(f"{key} = {value}")
+        elif isinstance(value, str):
+            native_conditions.append(f"{key} = '{_escape_zvec_string(value)}'")
+        else:
+            # Unsupported type — fall back to post-filtering
+            post_filters[key] = value
+
+    native_filter = " AND ".join(native_conditions) if native_conditions else None
+    return native_filter, post_filters if post_filters else None
+
+
+def _metric_type_from_str(metric: str) -> Any:
+    """Convert a string metric name to zvec MetricType enum value."""
+    if zvec is None:
+        return None
+    mapping = {
+        "cosine": MetricType.COSINE,
+        "l2": MetricType.L2,
+        "ip": MetricType.IP,
+    }
+    return mapping.get(metric.lower(), MetricType.COSINE)
+
+
+def _build_collection_schema(
+    name: str,
+    dimension: int,
+    metric: str = "cosine",
+) -> CollectionSchema:
+    """Build a zvec CollectionSchema for ReMe usage.
+
+    The schema contains:
+    - "content" (STRING, inverted index) — text content
+    - "metadata" (STRING) — JSON-serialized metadata dictionary
+    - Promoted metadata fields (STRING / INT64) — for native zvec filtering
+    - "embedding" (VECTOR_FP32, dimension, HNSW index) — the vector field
+
+    Promoted fields are commonly filtered metadata keys defined as top-level
+    schema columns so that zvec can perform filtering natively instead of
+    Python post-filtering.  The full metadata is still stored as JSON in the
+    "metadata" field for complete round-trip serialization.
+
+    zvec automatically manages the document ID (string type); we do NOT
+    define an "id" field in the schema.
+    """
+    # Populate the DataType map on first call
+    if not _DATATYPE_MAP:
+        _DATATYPE_MAP.update(
+            {
+                "STRING": DataType.STRING,
+                "INT64": DataType.INT64,
+            },
+        )
+
+    distance = _metric_type_from_str(metric)
+
+    # Base fields
+    fields = [
+        FieldSchema("content", DataType.STRING, nullable=True, index_param=InvertIndexParam()),
+        FieldSchema("metadata", DataType.STRING, nullable=True),
+    ]
+
+    # Add promoted metadata fields as top-level schema columns
+    for field_name, (type_str, has_inv_index) in _PROMOTED_FIELD_SPECS.items():
+        dt = _DATATYPE_MAP[type_str]
+        idx_param = InvertIndexParam() if has_inv_index else None
+        fields.append(FieldSchema(field_name, dt, nullable=True, index_param=idx_param))
+
+    return CollectionSchema(
+        name=name,
+        fields=fields,
+        vectors=[
+            VectorSchema(
+                name=_DEFAULT_VECTOR_FIELD,
+                data_type=DataType.VECTOR_FP32,
+                dimension=dimension,
+                index_param=HnswIndexParam(metric_type=distance),
+            ),
+        ],
+    )
+
+
+def _vector_node_to_doc(node: VectorNode) -> Doc:
+    """Convert a ReMe VectorNode to a zvec Doc.
+
+    Metadata is serialized as a JSON string into the "metadata" field.
+    The "score" key is excluded since it is a computed value, not stored data.
+    Promoted metadata fields are also extracted as top-level Doc fields
+    for native zvec filtering.
+    The vector is placed under the default vector field name.
+    The zvec Doc id must be a string.
+    """
+    # Filter out computed score before serialization
+    meta_to_store = {k: v for k, v in node.metadata.items() if k != "score"}
+
+    fields: dict[str, Any] = {
+        "content": node.content,
+        "metadata": json.dumps(meta_to_store) if meta_to_store else "{}",
+    }
+
+    # Extract promoted metadata fields as top-level schema columns
+    for field_name, (type_str, _) in _PROMOTED_FIELD_SPECS.items():
+        value = meta_to_store.get(field_name)
+        if value is not None:
+            # Ensure correct type: INT64 fields must be int
+            if type_str == "INT64" and not isinstance(value, int):
+                try:
+                    value = int(value)
+                except (ValueError, TypeError):
+                    continue
+            fields[field_name] = value
+
+    vectors: dict[str, Any] = {}
+    if node.vector is not None:
+        vectors[_DEFAULT_VECTOR_FIELD] = node.vector
+
+    return Doc(id=str(node.vector_id), fields=fields, vectors=vectors)
+
+
+def _doc_to_vector_node(doc: Doc, include_score: bool = False) -> VectorNode:
+    """Convert a zvec Doc back to a ReMe VectorNode.
+
+    The "metadata" field is parsed from JSON. The "content" field becomes
+    the node content.  If ``include_score`` is True, the search score is
+    added to the metadata dictionary.
+    """
+    metadata: dict[str, str | bool | int | float] = {}
+
+    # Parse JSON metadata
+    raw_metadata = doc.field("metadata")
+    if raw_metadata:
+        try:
+            parsed = json.loads(raw_metadata)
+            if isinstance(parsed, dict):
+                metadata.update(parsed)
+        except (json.JSONDecodeError, TypeError):
+            logger.warning(f"Failed to parse metadata JSON: {raw_metadata}")
+
+    if include_score and doc.score is not None:
+        metadata["score"] = doc.score
+
+    # Extract vector — doc.vector() returns list or empty dict
+    raw_vector = doc.vector(_DEFAULT_VECTOR_FIELD)
+    vector = raw_vector if isinstance(raw_vector, list) and len(raw_vector) > 0 else None
+
+    content = doc.field("content") or ""
+
+    return VectorNode(
+        vector_id=str(doc.id),
+        content=str(content),
+        vector=vector,
+        metadata=metadata,
+    )
+
+
+def _apply_filters_post(nodes: list[VectorNode], filters: dict | None) -> list[VectorNode]:
+    """Apply ReMe-style filter dict as post-filtering on metadata.
+
+    Used as a fallback for metadata keys that are NOT promoted to top-level
+    schema columns (and thus cannot be filtered natively by zvec).  Promoted
+    fields are handled by zvec's native ``filter`` parameter instead.
+
+    Supports:
+    - Exact match: {"field": value}
+    - Range query: {"field": [start, end]}
+    """
+    if not filters:
+        return nodes
+
+    filtered = []
+    for node in nodes:
+        match = True
+        for key, value in filters.items():
+            if key.startswith("$"):
+                # Skip compound operators for post-filtering
+                continue
+            node_value = node.metadata.get(key)
+
+            # Range query: [start, end]
+            if isinstance(value, list) and len(value) == 2:
+                if node_value is None:
+                    match = False
+                    break
+                try:
+                    if not value[0] <= node_value <= value[1]:
+                        match = False
+                        break
+                except TypeError:
+                    match = False
+                    break
+            else:
+                # Exact match
+                if node_value != value:
+                    match = False
+                    break
+
+        if match:
+            filtered.append(node)
+
+    return filtered
+
+
+class ZvecVectorStore(BaseVectorStore):
+    """Zvec-based vector store implementation.
+
+    Zvec is a high-performance vector database. This adapter bridges the
+    ReMe ``BaseVectorStore`` interface with zvec's Python API.
+
+    Supports local persistent storage via ``db_path``.
+
+    Args:
+        collection_name: Name of the vector collection.
+        db_path: Local storage path for persistent mode.
+        embedding_model: Model used for generating vector embeddings.
+        dimension: Dimensionality of the embedding vectors (default: 1024).
+        distance: Distance metric — cosine / l2 / ip (default: cosine).
+        **kwargs: Additional zvec-specific configuration.
+    """
+
+    def __init__(
+        self,
+        collection_name: str,
+        db_path: str | Path,
+        embedding_model: BaseEmbeddingModel,
+        dimension: int = 1024,
+        distance: str = "cosine",
+        **kwargs: Any,
+    ):
+        """Initialize the Zvec vector store."""
+        if _ZVEC_IMPORT_ERROR is not None:
+            raise ImportError(
+                "Zvec requires extra dependencies. Install with `pip install zvec`",
+            ) from _ZVEC_IMPORT_ERROR
+
+        super().__init__(
+            collection_name=collection_name,
+            db_path=db_path,
+            embedding_model=embedding_model,
+            **kwargs,
+        )
+
+        self.dimension = dimension
+        self.distance = distance
+        self._collection = None
+        self._initialized = False
+        # Set of promoted field names that exist in the current collection's schema.
+        # Populated during start() by inspecting the schema.  Only fields present
+        # in the schema can use native zvec filtering; the rest fall back to
+        # Python post-filtering.
+        self._promoted_fields_in_schema: set[str] = set()
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    async def start(self) -> None:
+        """Initialize the Zvec engine and open the collection.
+
+        Calls ``zvec.init()`` once, then tries to ``zvec.open()`` an existing
+        collection or ``zvec.create_and_open()`` a new one.
+        After opening, detects which promoted fields exist in the schema
+        and attempts to add missing numeric fields via ``add_column``.
+        """
+        if not self._initialized:
+            try:
+                zvec.init()
+            except RuntimeError:
+                # Already initialized — safe to ignore
+                pass
+            self._initialized = True
+
+        self.db_path.mkdir(parents=True, exist_ok=True)
+        collection_path = str(self.db_path / self.collection_name)
+
+        option = CollectionOption(read_only=False, enable_mmap=True)
+
+        try:
+            # Try opening an existing collection first
+            self._collection = zvec.open(collection_path, option)
+            logger.info(f"Opened existing Zvec collection at {collection_path}")
+        except Exception:
+            # Collection doesn't exist — create it
+            schema = _build_collection_schema(
+                name=self.collection_name,
+                dimension=self.dimension,
+                metric=self.distance,
+            )
+            self._collection = zvec.create_and_open(
+                path=collection_path,
+                schema=schema,
+                option=option,
+            )
+            logger.info(f"Created new Zvec collection at {collection_path}")
+
+        # Detect which promoted fields exist in the current schema
+        self._detect_promoted_fields()
+
+        # Try to add missing numeric promoted fields to existing collections
+        # (zvec's add_column only supports numeric types: INT64, FLOAT, etc.)
+        self._ensure_numeric_promoted_columns()
+
+    async def close(self) -> None:
+        """Flush pending writes and release the collection handle."""
+        if self._collection is not None:
+            try:
+                self._collection.flush()
+            except Exception as e:
+                logger.warning(f"Failed to flush collection on close: {e}")
+            self._collection = None
+        logger.info(f"Zvec vector store for collection {self.collection_name} closed")
+
+    # ------------------------------------------------------------------
+    # Collection management
+    # ------------------------------------------------------------------
+
+    async def list_collections(self) -> list[str]:
+        """Retrieve a list of collection names in the db_path directory.
+
+        Zvec doesn't have a global ``list_collections`` API; we scan the
+        db_path directory for zvec collection folders.
+        """
+        if not self.db_path.exists():
+            return []
+        collections = []
+        for child in self.db_path.iterdir():
+            if child.is_dir():
+                collections.append(child.name)
+        return collections
+
+    async def create_collection(self, collection_name: str, **kwargs) -> None:
+        """Create a new collection with the specified name and distance metric."""
+        if not self._initialized:
+            try:
+                zvec.init()
+            except RuntimeError:
+                pass
+            self._initialized = True
+
+        self.db_path.mkdir(parents=True, exist_ok=True)
+        collection_path = str(self.db_path / collection_name)
+
+        dimension = kwargs.get("dimension", self.dimension)
+        metric = kwargs.get("distance_metric", self.distance)
+
+        schema = _build_collection_schema(
+            name=collection_name,
+            dimension=dimension,
+            metric=metric,
+        )
+        option = CollectionOption(read_only=False, enable_mmap=True)
+
+        collection = zvec.create_and_open(path=collection_path, schema=schema, option=option)
+        if collection_name == self.collection_name:
+            self._collection = collection
+        logger.info(f"Created collection `{collection_name}`")
+
+    async def delete_collection(self, collection_name: str, **kwargs) -> None:
+        """Permanently remove a collection from disk."""
+        # If it's the active collection, destroy it via zvec API
+        if self._collection is not None and collection_name == self.collection_name:
+            try:
+                self._collection.destroy()
+                self._collection = None
+                deleted = True
+            except Exception as _e:
+                logger.warning(f"Failed to destroy collection {collection_name}: {_e}")
+                deleted = False
+        else:
+            # For non-active collections, remove the directory
+            collection_path = self.db_path / collection_name
+            if collection_path.exists():
+                import shutil
+
+                shutil.rmtree(collection_path, ignore_errors=True)
+                deleted = True
+            else:
+                deleted = False
+
+        logger.info(f"Deleted collection {collection_name}: {deleted}")
+
+    async def copy_collection(self, collection_name: str, **kwargs) -> None:
+        """Duplicate the current collection to a new one with the given name.
+
+        Uses ``shutil.copytree`` to directly copy the collection directory on
+        disk, which is both faster and complete — it avoids the topk limit of
+        ``list()`` (max 1024 docs) that would cause data loss for large
+        collections.
+
+        The source collection is flushed before copying to ensure all
+        pending writes are persisted to disk.
+        """
+        import shutil
+
+        # Flush source collection so all data is on disk
+        if self._collection is not None:
+            self._collection.flush()
+
+        src_path = self.db_path / self.collection_name
+        dst_path = self.db_path / collection_name
+
+        if not src_path.exists():
+            logger.warning(f"Source collection directory not found: {src_path}")
+            return
+
+        if dst_path.exists():
+            logger.warning(f"Target collection already exists: {dst_path}, removing it first")
+            shutil.rmtree(dst_path, ignore_errors=True)
+
+        shutil.copytree(src_path, dst_path)
+        logger.info(
+            f"Copied collection {self.collection_name} to {collection_name} "
+            f"(directory copy: {src_path} -> {dst_path})",
+        )
+
+    # ------------------------------------------------------------------
+    # CRUD operations
+    # ------------------------------------------------------------------
+
+    async def insert(self, nodes: VectorNode | list[VectorNode], **kwargs) -> None:
+        """Add one or more vector nodes into the current collection.
+
+        Automatically generates embeddings for nodes that lack vectors.
+        """
+        if isinstance(nodes, VectorNode):
+            nodes = [nodes]
+        if not nodes:
+            return
+
+        # Batch generate embeddings for nodes that need them
+        nodes_without_vectors = [n for n in nodes if n.vector is None]
+        if nodes_without_vectors:
+            nodes_with_vectors = await self.get_node_embeddings(nodes_without_vectors)
+            vector_map = {n.vector_id: n for n in nodes_with_vectors}
+            nodes_to_insert = [vector_map.get(n.vector_id, n) if n.vector is None else n for n in nodes]
+        else:
+            nodes_to_insert = nodes
+
+        batch_size = kwargs.get("batch_size", 100)
+
+        for i in range(0, len(nodes_to_insert), batch_size):
+            batch = nodes_to_insert[i : i + batch_size]
+            docs = [_vector_node_to_doc(n) for n in batch]
+            self._collection.insert(docs)
+
+        logger.info(f"Inserted {len(nodes_to_insert)} nodes into {self.collection_name}")
+
+    async def search(
+        self,
+        query: str,
+        limit: int = 5,
+        filters: dict | None = None,
+        **kwargs,
+    ) -> list[VectorNode]:
+        """Find the most similar vector nodes based on a text query.
+
+        Uses zvec's ``query()`` method with a ``VectorQuery`` built from the
+        embedding of the query text.  Promoted metadata fields are filtered
+        natively via zvec's ``filter`` parameter; remaining filters are
+        applied as post-filtering in Python.
+        """
+        query_vector = await self.get_embedding(query)
+
+        vq = VectorQuery(
+            field_name=_DEFAULT_VECTOR_FIELD,
+            vector=query_vector,
+        )
+        include_vector = kwargs.get("include_embeddings", False)
+
+        # Split filters: native zvec filter vs Python post-filter
+        native_filter, post_filters = _build_zvec_filter(filters, self._promoted_fields_in_schema)
+
+        # Over-fetch to compensate for post-filtering
+        _ZVEC_MAX_TOPK = 1024
+        # When post-filters remain, we need to fetch more results because
+        # many may be filtered out. Use the maximum allowed to minimize misses.
+        fetch_limit = _ZVEC_MAX_TOPK if post_filters else min(limit, _ZVEC_MAX_TOPK)
+
+        results = self._collection.query(
+            vectors=vq,
+            topk=fetch_limit,
+            filter=native_filter,
+            include_vector=include_vector,
+        )
+
+        nodes = [_doc_to_vector_node(doc, include_score=True) for doc in results]
+
+        # Post-filter on non-promoted metadata fields
+        nodes = _apply_filters_post(nodes, post_filters)
+
+        score_threshold = kwargs.get("score_threshold")
+        if score_threshold is not None:
+            nodes = [n for n in nodes if n.metadata.get("score", 0) >= score_threshold]
+
+        return nodes[:limit]
+
+    async def delete(self, vector_ids: str | list[str], **kwargs) -> None:
+        """Remove specific vectors from the collection using their identifiers."""
+        if isinstance(vector_ids, str):
+            vector_ids = [vector_ids]
+        if not vector_ids:
+            return
+
+        self._collection.delete(vector_ids)
+        logger.info(f"Deleted {len(vector_ids)} nodes from {self.collection_name}")
+
+    async def delete_all(self, **kwargs) -> None:
+        """Remove all vectors from the collection.
+
+        Uses zvec's ``delete_by_filter`` with a condition that matches all
+        documents (content is not empty), or falls back to query + delete
+        in batches (zvec topk max is 1024).
+        """
+        stats = self._collection.stats
+        count = stats.doc_count if stats else 0
+        if count > 0:
+            try:
+                # Use delete_by_filter for efficiency
+                self._collection.delete_by_filter("content!=''")
+            except Exception:
+                # Fallback: fetch all IDs in batches then delete
+                _ZVEC_MAX_TOPK = 1024
+                remaining = count
+                while remaining > 0:
+                    all_docs = self._collection.query(topk=min(remaining, _ZVEC_MAX_TOPK), include_vector=False)
+                    if not all_docs:
+                        break
+                    ids = [doc.id for doc in all_docs]
+                    self._collection.delete(ids)
+                    remaining -= len(ids)
+        logger.info(f"Deleted all {count} nodes from {self.collection_name}")
+
+    async def update(self, nodes: VectorNode | list[VectorNode], **kwargs) -> None:
+        """Update existing vectors using zvec's ``upsert``.
+
+        Automatically regenerates embeddings for nodes whose content changed
+        but lack an updated vector.
+        """
+        if isinstance(nodes, VectorNode):
+            nodes = [nodes]
+        if not nodes:
+            return
+
+        # Batch generate embeddings for nodes that need them
+        nodes_without_vectors = [n for n in nodes if n.vector is None and n.content]
+        if nodes_without_vectors:
+            nodes_with_vectors = await self.get_node_embeddings(nodes_without_vectors)
+            vector_map = {n.vector_id: n for n in nodes_with_vectors}
+            nodes_to_update = [vector_map.get(n.vector_id, n) if n.vector is None and n.content else n for n in nodes]
+        else:
+            nodes_to_update = nodes
+
+        docs = [_vector_node_to_doc(n) for n in nodes_to_update]
+        self._collection.upsert(docs)
+        logger.info(f"Updated {len(nodes_to_update)} nodes in {self.collection_name}")
+
+    async def get(self, vector_ids: str | list[str]) -> VectorNode | list[VectorNode]:
+        """Fetch specific vector nodes from the collection by their IDs."""
+        is_single = isinstance(vector_ids, str)
+        ids = [vector_ids] if is_single else vector_ids
+
+        result_dict = self._collection.fetch(ids)
+        nodes = [_doc_to_vector_node(doc) for doc in result_dict.values()]
+        return nodes[0] if is_single and nodes else (nodes if not is_single else None)
+
+    async def list(
+        self,
+        filters: dict | None = None,
+        limit: int | None = None,
+        sort_key: str | None = None,
+        reverse: bool = True,
+    ) -> list[VectorNode]:
+        """Retrieve vectors matching optional metadata filters.
+
+        Uses zvec's ``query()`` without a vector query to list all documents.
+        Promoted metadata fields are filtered natively via zvec's ``filter``
+        parameter; remaining filters are applied as post-filtering in Python.
+
+        Args:
+            filters: Dictionary of filter conditions to match vectors.
+            limit: Maximum number of vectors to return.
+            sort_key: Key to sort the results by (in metadata).
+            reverse: If True, sort in descending order; otherwise ascending.
+        """
+        # Split filters: native zvec filter vs Python post-filter
+        native_filter, post_filters = _build_zvec_filter(filters, self._promoted_fields_in_schema)
+
+        # Determine fetch limit — zvec max topk is 1024 (will be lifted to 100,000 in zvec v0.3.2+)
+        _ZVEC_MAX_TOPK = 1024
+        fetch_limit = min(limit or _ZVEC_MAX_TOPK, _ZVEC_MAX_TOPK)
+        if sort_key or post_filters:
+            fetch_limit = _ZVEC_MAX_TOPK  # fetch max and sort/filter in Python
+
+        results = self._collection.query(
+            topk=fetch_limit,
+            filter=native_filter,
+            include_vector=True,
+        )
+
+        nodes = [_doc_to_vector_node(doc) for doc in results]
+
+        # Post-filter on non-promoted metadata fields
+        nodes = _apply_filters_post(nodes, post_filters)
+
+        # Apply sorting if sort_key is provided
+        if sort_key:
+
+            def _sort_key_func(node: VectorNode):
+                value = node.metadata.get(sort_key)
+                if value is None:
+                    return float("-inf") if not reverse else float("inf")
+                return value
+
+            nodes.sort(key=_sort_key_func, reverse=reverse)
+
+        if limit is not None:
+            nodes = nodes[:limit]
+
+        return nodes
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    def _detect_promoted_fields(self) -> None:
+        """Detect which promoted fields exist in the current collection's schema.
+
+        Compares the set of promoted field names against the actual schema
+        and populates ``_promoted_fields_in_schema`` accordingly.  Only fields
+        present in the schema can use native zvec filtering.
+        """
+        if self._collection is None:
+            return
+
+        try:
+            schema = self._collection.schema
+            existing_fields = {f.name for f in schema.fields} if schema.fields else set()
+        except Exception as e:
+            logger.warning(f"Failed to read collection schema: {e}")
+            existing_fields = set()
+
+        self._promoted_fields_in_schema = set(_PROMOTED_FIELD_SPECS.keys()) & existing_fields
+
+        missing = set(_PROMOTED_FIELD_SPECS.keys()) - existing_fields
+        if missing:
+            logger.info(
+                f"Promoted fields not in schema (will use post-filtering): {missing}",
+            )
+
+    def _ensure_numeric_promoted_columns(self) -> None:
+        """Add missing numeric promoted fields to existing collections.
+
+        zvec's ``add_column`` only supports numeric types (INT64, FLOAT, etc.).
+        STRING fields cannot be added via ``add_column`` and must be defined
+        at collection creation time.  For those, we fall back to post-filtering.
+        """
+        if self._collection is None:
+            return
+
+        missing = set(_PROMOTED_FIELD_SPECS.keys()) - self._promoted_fields_in_schema
+        if not missing:
+            return
+
+        # Populate the DataType map if needed
+        if not _DATATYPE_MAP:
+            _DATATYPE_MAP.update(
+                {
+                    "STRING": DataType.STRING,
+                    "INT64": DataType.INT64,
+                },
+            )
+
+        for field_name in missing:
+            type_str, _ = _PROMOTED_FIELD_SPECS[field_name]
+            # Only numeric types can be added via add_column
+            if type_str not in ("INT64", "INT32", "FLOAT", "DOUBLE"):
+                continue
+            try:
+                dt = _DATATYPE_MAP[type_str]
+                self._collection.add_column(FieldSchema(field_name, dt, nullable=True))
+                self._promoted_fields_in_schema.add(field_name)
+                logger.info(f"Added promoted column '{field_name}' to existing collection")
+            except Exception as e:
+                logger.warning(f"Failed to add column '{field_name}': {e}")
+
+    async def count(self) -> int:
+        """Return the total number of documents in the current collection."""
+        stats = self._collection.stats
+        return stats.doc_count if stats else 0
+
+    async def reset(self):
+        """Reset the current collection by destroying and recreating it."""
+        logger.warning(f"Resetting collection {self.collection_name}...")
+        await self.delete_collection(self.collection_name)
+        await self.create_collection(self.collection_name)
+        logger.info(f"Collection {self.collection_name} has been reset")

reme/memory/vector_based/personal/personal_retriever.py

@@ -1,39 +1,19 @@
 """Personal memory retriever agent for retrieving personal memories through vector search."""
 
+from loguru import logger
+
 from ..base_memory_agent import BaseMemoryAgent
-from ....core.enumeration import Role, MemoryType
+from ....core.enumeration import MemoryType, Role
 from ....core.op import BaseTool
 from ....core.schema import Message
 from ....core.utils import format_messages
 
+_PROFILE_TOOL_NAMES: tuple[str, ...] = ("retrieve_profile", "read_all_profiles")
+_EMPTY_PROFILE_RESULTS: tuple[str, ...] = ("", "No profiles found.", "No new profiles found.")
+
 
 class PersonalRetriever(BaseMemoryAgent):
-    """Retrieve personal memories through vector search and history reading.
-
-    clear && python benchmark/halumem/eval_reme.py \
-        --data_path /Users/yuli/workspace/HaluMem/data/HaluMem-Medium.jsonl \
-        --reme_model_name qwen3.5-plus \
-        --batch_size 10000 \
-        --algo_version default
-
-    📊 Question Answering (with LLM answer):
-      Correct (all):       0.8537
-      Hallucination (all): 0.1159
-      Omission (all):      0.0305
-      Correct (valid):     0.8537
-      Hallucination (valid): 0.1159
-      Omission (valid):    0.0305
-      Valid/Total:         164/164
-
-    📊 Question Answering (with original memories):
-      Correct (all):       0.9085
-      Hallucination (all): 0.0671
-      Omission (all):      0.0244
-      Correct (valid):     0.9085
-      Hallucination (valid): 0.0671
-      Omission (valid):    0.0244
-      Valid/Total:         164/164
-    """
+    """Retrieve personal memories through vector search and history reading."""
 
     memory_type: MemoryType = MemoryType.PERSONAL
 
@@ -41,36 +21,73 @@ class PersonalRetriever(BaseMemoryAgent):
         super().__init__(**kwargs)
         self.return_memory_nodes: bool = return_memory_nodes
 
-    async def build_messages(self) -> list[Message]:
+    def _get_context(self) -> str:
         if self.context.get("query"):
-            context = self.context.query
-        elif self.context.get("messages"):
-            context = self.description + "\n" + format_messages(self.context.messages)
-        else:
-            raise ValueError("input must have either `query` or `messages`")
-
-        read_all_profiles_tool: BaseTool | None = self.pop_tool("read_all_profiles")
-        if read_all_profiles_tool is not None:
-            all_profiles = await read_all_profiles_tool.call(
-                memory_target=self.memory_target,
-                service_context=self.service_context,
-            )
-        else:
-            all_profiles = ""
+            return self.context.query.strip()
+        if self.context.get("messages"):
+            return (self.description + "\n" + format_messages(self.context.messages)).strip()
+        raise ValueError("input must have either `query` or `messages`")
 
+    async def _build_s1_messages(self, context: str) -> list[Message]:
         return [
             Message(
                 role=Role.USER,
                 content=self.prompt_format(
-                    prompt_name="user_message",
+                    prompt_name="user_message_s1",
                     memory_type=self.memory_type.value,
                     memory_target=self.memory_target,
-                    user_profile=all_profiles,
-                    context=context.strip(),
+                    context=context,
                 ),
             ),
         ]
 
+    async def _build_s2_messages(self, context: str, profiles: str) -> list[Message]:
+        return [
+            Message(
+                role=Role.USER,
+                content=self.prompt_format(
+                    prompt_name="user_message_s2",
+                    memory_type=self.memory_type.value,
+                    memory_target=self.memory_target,
+                    profiles=profiles,
+                    context=context,
+                ),
+            ),
+        ]
+
+    def _partition_tools(self) -> tuple[list[BaseTool], list[BaseTool]]:
+        profile_tools: list[BaseTool] = []
+        memory_tools: list[BaseTool] = []
+        for i, tool in enumerate(self.tools):
+            name = tool.tool_call.name
+            if name in _PROFILE_TOOL_NAMES:
+                profile_tools.append(tool)
+            else:
+                memory_tools.append(tool)
+            logger.info(f"[{self.__class__.__name__}] tool_call[{i}]={tool.tool_call.simple_input_dump(as_dict=False)}")
+        return profile_tools, memory_tools
+
+    @staticmethod
+    def _extract_profile_context(tools: list[BaseTool]) -> str:
+        outputs = []
+        for tool in tools:
+            response = getattr(tool, "response", None)
+            answer = getattr(response, "answer", "")
+            if answer and answer not in _EMPTY_PROFILE_RESULTS:
+                outputs.append(answer)
+        return "\n".join(outputs)
+
+    async def _run_stage(
+        self,
+        stage: str,
+        messages: list[Message],
+        tools: list[BaseTool],
+    ) -> tuple[list[BaseTool], list[Message], bool]:
+        for message in messages:
+            role = message.name or message.role
+            logger.info(f"[{self.__class__.__name__} {stage}] role={role} {message.simple_dump(as_dict=False)}")
+        return await self.react(messages, tools, stage=stage)
+
     async def _acting_step(
         self,
         assistant_message: Message,
@@ -91,7 +108,28 @@ class PersonalRetriever(BaseMemoryAgent):
         )
 
     async def execute(self):
-        result = await super().execute()
+        context = self._get_context()
+        profile_tools, memory_tools = self._partition_tools()
+
+        tools_s1: list[BaseTool] = []
+        messages_s1: list[Message] = []
+        success_s1 = True
+        profiles = ""
+        if profile_tools:
+            messages_s1 = await self._build_s1_messages(context)
+            tools_s1, messages_s1, success_s1 = await self._run_stage("s1-profile", messages_s1, profile_tools)
+            profiles = self._extract_profile_context(tools_s1)
+
+        messages_s2 = await self._build_s2_messages(context, profiles)
+        tools_s2, messages_s2, success_s2 = await self._run_stage("s2-memory", messages_s2, memory_tools)
+
+        answer = messages_s2[-1].content if success_s2 and messages_s2 else ""
+        result = {
+            "answer": answer,
+            "success": success_s1 and success_s2,
+            "messages": messages_s1 + messages_s2,
+            "tools": tools_s1 + tools_s2,
+        }
         if self.return_memory_nodes:
             result["answer"] = "\n".join(
                 [

reme/memory/vector_based/personal/personal_retriever.yaml

@@ -1,8 +1,26 @@
-user_message: |
+user_message_s1: |
+  You are a Profile Retrieval Agent specialized in finding profile information about {memory_target}.
+
+  ## User Question
+  {context}
+
+  ## Task
+  Use the available profile tool to search for profile content that is relevant to the user question.
+
+  ## Instructions
+  - If `retrieve_profile` is available, use it to search with focused profile queries derived from the question
+  - If `read_all_profiles` is available, use it to inspect the full profile list and identify relevant rows
+  - Focus on profile attributes such as identity, location, work, education, preferences, relationships, and other long-term facts
+  - Only retrieve information that is directly relevant to the user question
+  - If no relevant profile information exists, say so clearly
+
+  Output a concise summary of the relevant profile information you found.
+
+user_message_s2: |
   You are a Memory Retrieval Agent specialized in retrieving {memory_type} memories about {memory_target}.
 
-  ## User Profile
-  {user_profile}
+  ## Profile Search Results
+  {profiles}
 
   ## User Question
   {context}
@@ -14,11 +32,12 @@ user_message: |
   **Tool**: `retrieve_memory` (without time constraints)
   **Objective**: Cast a wide net to find potentially relevant memories
   **Approach**:
+  - Use the profile search results above as supporting context when forming retrieval queries
   - Execute 3-5 diverse search queries using different formulations:
     * Original question verbatim
     * Rephrased variations (different wording, synonyms)
     * Entity-focused queries (extract and search specific names, places, events)
-    * Keyword-based searches (core concepts, topics)
+    * Keyword-based searches (core concepts and profile facts)
     * Related context queries (broader themes)
 
   ### Phase 2(Optional): Temporal Search
@@ -31,7 +50,7 @@ user_message: |
   - After date: `20200101,99999999` (from 20200101 onwards)
   **Approach**:
   - Identify temporal constraints from the user question
-  - Refine Phase 1 queries with 3-5 diverse appropriate different time filters
+  - Refine Phase 1 queries with 3-5 diverse appropriate time filters
 
   ### Phase 3: Deep Dive into History
   **Tool**: `read_history`
@@ -48,11 +67,11 @@ user_message: |
   - Use this to understand the full conversation surrounding a memory
 
   ## Response Guidelines
-  - Base your answer EXCLUSIVELY on user profile, retrieved memories, and history data
+  - Base your answer EXCLUSIVELY on the profile search results, retrieved memories, and history data
   - Never infer, assume, or hallucinate information
   - Always cite sources with timestamps: `[timestamp] Memory content`
   - Present conflicting information transparently with respective timestamps
   - If you find sufficient information to answer the user's question, you may output directly without exhausting all search phases
   - Exhaust all search strategies before concluding information doesn't exist
 
-  Output a summary of all retrieved memories, user profile, and history data.
+  Output a summary of all retrieved memories and relevant history data.

reme/memory/vector_based/personal/personal_summarizer.py

@@ -3,13 +3,17 @@
 from loguru import logger
 
 from ..base_memory_agent import BaseMemoryAgent
-from ....core.enumeration import Role, MemoryType
+from ....core.enumeration import MemoryType, Role
 from ....core.op import BaseTool
 from ....core.schema import Message
 
+# Optional profile tools used to pre-load profile context; consumed by the
+# summarizer itself and never exposed to the stage-two ReAct loop.
+_PROFILE_CONTEXT_TOOLS: tuple[str, ...] = ("retrieve_profile", "read_all_profiles")
+
 
 class PersonalSummarizer(BaseMemoryAgent):
-    """Two-phase personal memory processor: retrieve/add memories then update profile."""
+    """Two-phase personal memory processor: add memories, then update profiles."""
 
     memory_type: MemoryType = MemoryType.PERSONAL
 
@@ -62,62 +66,71 @@ class PersonalSummarizer(BaseMemoryAgent):
             **kwargs,
         )
 
-    async def execute(self):
-        memory_tools = []
-        profile_tools = []
-        read_all_profiles_tool: BaseTool | None = None
+    def _partition_tools(self) -> tuple[list[BaseTool], list[BaseTool], BaseTool | None]:
+        """Split attached tools into memory tools, profile tools, and a profile context tool."""
+        memory_tools: list[BaseTool] = []
+        profile_tools: list[BaseTool] = []
+        profile_context_tool: BaseTool | None = None
         for i, tool in enumerate(self.tools):
-            tool_name = tool.tool_call.name
-            if tool_name == "read_all_profiles":
-                read_all_profiles_tool = tool
-            elif "_memory" in tool_name:
+            name = tool.tool_call.name
+            if name in _PROFILE_CONTEXT_TOOLS:
+                profile_context_tool = tool
+            elif "_memory" in name:
                 memory_tools.append(tool)
-            elif "_profile" in tool_name:
+            elif "_profile" in name:
                 profile_tools.append(tool)
             else:
-                raise ValueError(f"[{self.__class__.__name__}] unknown tool_name={tool_name}")
+                raise ValueError(f"[{self.__class__.__name__}] unknown tool_name={name}")
             logger.info(f"[{self.__class__.__name__}] tool_call[{i}]={tool.tool_call.simple_input_dump(as_dict=False)}")
+        return memory_tools, profile_tools, profile_context_tool
 
-        stage = "s1-memory"
-        messages_s1 = await self._build_s1_messages()
-        for i, message in enumerate(messages_s1):
+    async def _preload_user_profile(self, tool: BaseTool | None) -> str:
+        """Invoke the profile context tool to obtain inline profile text."""
+        if tool is None:
+            return ""
+        call_kwargs: dict = {
+            "memory_target": self.memory_target,
+            "service_context": self.service_context,
+            "retrieved_nodes": self.retrieved_nodes,
+        }
+        if tool.tool_call.name == "retrieve_profile":
+            call_kwargs["query"] = self.context.history_node.content
+        return await tool.call(**call_kwargs)
+
+    async def _run_stage(
+        self,
+        stage: str,
+        messages: list[Message],
+        tools: list[BaseTool],
+    ) -> tuple[list[BaseTool], list[Message], bool]:
+        for message in messages:
             role = message.name or message.role
             logger.info(f"[{self.__class__.__name__} {stage}] role={role} {message.simple_dump(as_dict=False)}")
-        tools_s1, messages_s1, success_s1 = await self.react(messages_s1, memory_tools, stage=stage)
+        return await self.react(messages, tools, stage=stage)
 
-        if read_all_profiles_tool is not None:
-            profiles = await read_all_profiles_tool.call(
-                memory_target=self.memory_target,
-                service_context=self.service_context,
-            )
-        else:
-            profiles = ""
+    async def execute(self):
+        memory_tools, profile_tools, profile_context_tool = self._partition_tools()
+
+        messages_s1 = await self._build_s1_messages()
+        tools_s1, messages_s1, success_s1 = await self._run_stage("s1-memory", messages_s1, memory_tools)
 
         if profile_tools:
-            stage = "s2-profile"
+            profiles = await self._preload_user_profile(profile_context_tool)
             messages_s2 = await self._build_s2_messages(profiles)
-            for i, message in enumerate(messages_s2):
-                role = message.name or message.role
-                logger.info(f"[{self.__class__.__name__} {stage}] role={role} {message.simple_dump(as_dict=False)}")
-            tools_s2, messages_s2, success_s2 = await self.react(messages_s2, profile_tools, stage=stage)
+            tools_s2, messages_s2, success_s2 = await self._run_stage("s2-profile", messages_s2, profile_tools)
         else:
             tools_s2, messages_s2, success_s2 = [], [], True
 
         answer = (messages_s1[-1].content if success_s1 and messages_s1 else "") + (
             messages_s2[-1].content if success_s2 and messages_s2 else ""
         )
-        success = success_s1 and success_s2
-        messages = messages_s1 + messages_s2
         tools = tools_s1 + tools_s2
-        memory_nodes = []
-        for tool in tools:
-            if tool.memory_nodes:
-                memory_nodes.extend(tool.memory_nodes)
+        memory_nodes = [node for tool in tools for node in (tool.memory_nodes or [])]
 
         return {
             "answer": answer,
-            "success": success,
-            "messages": messages,
+            "success": success_s1 and success_s2,
+            "messages": messages_s1 + messages_s2,
             "tools": tools,
             "memory_nodes": memory_nodes,
         }

reme/memory/vector_tools/__init__.py

@@ -1,5 +1,7 @@
 """memory tools"""
 
+# pylint: disable=no-name-in-module
+
 from .base_memory_tool import BaseMemoryTool
 
 # chunk tools
@@ -15,6 +17,7 @@ from .profiles.add_draft_and_read_all_profiles import AddDraftAndReadAllProfiles
 from .profiles.add_profile import AddProfile
 from .profiles.delete_profile import DeleteProfile
 from .profiles.read_all_profiles import ReadAllProfiles
+from .profiles.retrieve_profile import RetrieveProfile
 from .profiles.update_profile import UpdateProfile
 from .profiles.update_profiles_v1 import UpdateProfilesV1
 
@@ -43,6 +46,7 @@ __all__ = [
     "AddProfile",
     "DeleteProfile",
     "ReadAllProfiles",
+    "RetrieveProfile",
     "UpdateProfile",
     "UpdateProfilesV1",
     # record tools

reme/memory/vector_tools/base_memory_tool.py

@@ -3,6 +3,7 @@
 from abc import ABCMeta
 from pathlib import Path
 
+from .profiles.profile_handler import ProfileHandler
 from ...core.enumeration import MemoryType
 from ...core.op import BaseTool
 from ...core.schema import ToolCall, MemoryNode, ToolAttr
@@ -16,12 +17,18 @@ class BaseMemoryTool(BaseTool, metaclass=ABCMeta):
         enable_multiple: bool = True,
         enable_thinking_params: bool = False,
         profile_dir: str = "",
+        profile_backend: str = "filesystem",
+        profile_store_name: str = "profile",
+        profile_max_capacity: int = 50,
         **kwargs,
     ):
         super().__init__(**kwargs)
         self.enable_multiple: bool = enable_multiple
         self.enable_thinking_params: bool = enable_thinking_params
         self.profile_dir: str = profile_dir
+        self.profile_backend: str = profile_backend
+        self.profile_store_name: str = profile_store_name
+        self.profile_max_capacity: int = profile_max_capacity
 
     def _build_tool_call(self) -> ToolCall:
         """Build and return the tool call schema"""
@@ -103,6 +110,19 @@ class BaseMemoryTool(BaseTool, metaclass=ABCMeta):
         return self.context.service_context.memory_target_type_mapping
 
     @property
-    def profile_path(self) -> Path:
+    def profile_path(self) -> Path | None:
         """Get the path to the profile directory for the current collection."""
+        if not self.profile_dir:
+            return None
         return Path(self.profile_dir) / self.vector_store.collection_name
+
+    def get_profile_handler(self, memory_target: str) -> ProfileHandler:
+        """Build a profile handler for the current backend configuration."""
+        return ProfileHandler(
+            memory_target=memory_target,
+            profile_path=self.profile_path,
+            service_context=self.service_context,
+            profile_backend=self.profile_backend,
+            profile_store_name=self.profile_store_name,
+            max_capacity=self.profile_max_capacity,
+        )

reme/memory/vector_tools/profiles/__init__.py

@@ -0,0 +1 @@
+"""Profile memory tools."""

reme/memory/vector_tools/profiles/add_draft_and_read_all_profiles.py

@@ -1,8 +1,7 @@
-"""Add draft profile and read all profiles from local storage"""
+"""Add draft profile and read all profiles from the configured backend."""
 
 from loguru import logger
 
-from .profile_handler import ProfileHandler
 from ..base_memory_tool import BaseMemoryTool
 from ....core.schema import ToolCall
 
@@ -92,9 +91,8 @@ class AddDraftAndReadAllProfiles(BaseMemoryTool):
                 continue
             targets_processed.add(target)
 
-            profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=target)
-
-            profiles_str = profile_handler.read_all(add_profile_id=True)
+            profile_handler = self.get_profile_handler(target)
+            profiles_str = await profile_handler.aread_all(add_profile_id=True)
             if profiles_str:
                 all_profiles.append(f"## Profiles for {target}:\n{profiles_str}")
 

reme/memory/vector_tools/profiles/add_profile.py

@@ -1,8 +1,7 @@
-"""Add user profile tool"""
+"""Add user profile tool."""
 
 from loguru import logger
 
-from .profile_handler import ProfileHandler
 from ..base_memory_tool import BaseMemoryTool
 from ....core.schema import ToolCall
 
@@ -40,7 +39,7 @@ class AddProfile(BaseMemoryTool):
         )
 
     async def execute(self):
-        profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=self.memory_target)
+        profile_handler = self.get_profile_handler(self.memory_target)
 
         # Get parameters
         message_time = self.context.get("message_time", "")
@@ -58,7 +57,7 @@ class AddProfile(BaseMemoryTool):
         }
 
         # Add profile using ProfileHandler
-        new_nodes = profile_handler.add_batch(profiles=[profile], ref_memory_id=self.history_id)
+        new_nodes = await profile_handler.aadd_batch(profiles=[profile], ref_memory_id=self.history_id)
         self.memory_nodes.extend(new_nodes)
 
         if new_nodes:

reme/memory/vector_tools/profiles/delete_profile.py

@@ -1,8 +1,7 @@
-"""Delete user profile tool"""
+"""Delete user profile tool."""
 
 from loguru import logger
 
-from .profile_handler import ProfileHandler
 from ..base_memory_tool import BaseMemoryTool
 from ....core.schema import ToolCall
 
@@ -32,7 +31,7 @@ class DeleteProfile(BaseMemoryTool):
         )
 
     async def execute(self):
-        profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=self.memory_target)
+        profile_handler = self.get_profile_handler(self.memory_target)
 
         # Get profile_id parameter
         profile_id = self.context.get("profile_id", "")
@@ -41,7 +40,7 @@ class DeleteProfile(BaseMemoryTool):
             return "No profile_id provided, operation cancelled."
 
         # Delete profile using ProfileHandler
-        success = profile_handler.delete(profile_id)
+        success = await profile_handler.adelete(profile_id)
 
         if success:
             output = f"Successfully deleted profile with ID: {profile_id}"

reme/memory/vector_tools/profiles/file_profile_backend.py

@@ -0,0 +1,234 @@
+"""Filesystem-backed profile storage."""
+
+from pathlib import Path
+
+from loguru import logger
+
+from .profile_backend import BaseProfileBackend
+from ....core.enumeration import MemoryType
+from ....core.schema import MemoryNode
+from ....core.utils import CacheHandler, deduplicate_memories
+
+
+class FileProfileBackend(BaseProfileBackend):
+    """Persist user profiles in local JSONL cache files."""
+
+    def __init__(self, profile_path: str | Path, memory_target: str, max_capacity: int = 50):
+        super().__init__(memory_target=memory_target, max_capacity=max_capacity)
+        self.cache_key: str = self.memory_target.replace(" ", "_").lower()
+        self.cache_handler: CacheHandler = CacheHandler(profile_path)
+
+    def _load_nodes(self) -> list[MemoryNode]:
+        cached_data = self.cache_handler.load(self.cache_key, auto_clean=False)
+        if not cached_data:
+            return []
+        return [MemoryNode(**data) for data in cached_data]
+
+    def _save_nodes(self, nodes: list[MemoryNode], apply_limits: bool = True):
+        if apply_limits:
+            nodes = deduplicate_memories(nodes)
+
+            if len(nodes) > self.max_capacity:
+                sorted_nodes = sorted(nodes, key=lambda n: n.message_time)
+                removed_count = len(sorted_nodes) - self.max_capacity
+                nodes = sorted_nodes[removed_count:]
+                logger.info(
+                    f"Capacity limit reached: removed {removed_count} oldest profiles "
+                    f"(kept {len(nodes)}/{self.max_capacity})",
+                )
+
+        nodes_data = [node.model_dump(exclude_none=True) for node in nodes]
+        self.cache_handler.save(self.cache_key, nodes_data)
+        logger.info(f"Saved {len(nodes)} profiles to {self.cache_key}")
+
+    def get_all_sync(self) -> list[MemoryNode]:
+        """Load all profile nodes from cache, ordered by ``message_time``."""
+        nodes = self._load_nodes()
+        nodes.sort(key=lambda n: n.message_time)
+        return nodes
+
+    def get_by_sync(self, *, profile_id: str | None = None, profile_key: str | None = None) -> MemoryNode | None:
+        """Return the first node matching ``profile_id`` or ``profile_key``."""
+        if not profile_id and not profile_key:
+            raise ValueError("Must provide either profile_id or profile_key")
+
+        for node in self._load_nodes():
+            if profile_id and node.memory_id == profile_id:
+                return node
+            if profile_key and node.when_to_use == profile_key:
+                return node
+        return None
+
+    def delete_sync(self, profile_id: str | list[str]) -> bool | int:
+        """Remove one id, many ids, or none; returns bool, count, or 0/false if nothing removed."""
+        nodes = self._load_nodes()
+        original_count = len(nodes)
+
+        if isinstance(profile_id, list):
+            profile_ids_set = set(profile_id)
+            nodes = [n for n in nodes if n.memory_id not in profile_ids_set]
+            deleted_count = original_count - len(nodes)
+            if deleted_count == 0:
+                logger.warning(f"No profiles found to delete from {len(profile_id)} IDs")
+                return 0
+
+            self._save_nodes(nodes, apply_limits=False)
+            logger.info(f"Batch deleted {deleted_count} profiles")
+            return deleted_count
+
+        nodes = [n for n in nodes if n.memory_id != profile_id]
+        if len(nodes) == original_count:
+            logger.warning(f"Profile {profile_id} not found")
+            return False
+
+        self._save_nodes(nodes, apply_limits=False)
+        logger.info(f"Deleted profile {profile_id}")
+        return True
+
+    def delete_all_sync(self) -> int:
+        """Clear every cached profile for this target; returns how many were stored."""
+        nodes = self._load_nodes()
+        count = len(nodes)
+        self._save_nodes([], apply_limits=False)
+        logger.info(f"Deleted all {count} profiles")
+        return count
+
+    def add_sync(self, message_time: str, profile_key: str, profile_value: str, ref_memory_id: str = "") -> MemoryNode:
+        """Append a profile row, replacing any existing row with the same key."""
+        nodes = self._load_nodes()
+
+        new_node = MemoryNode(
+            memory_type=MemoryType.PERSONAL,
+            memory_target=self.memory_target,
+            when_to_use=profile_key,
+            content=profile_value,
+            message_time=message_time,
+            ref_memory_id=ref_memory_id,
+        )
+
+        original_count = len(nodes)
+        nodes = [n for n in nodes if n.when_to_use != profile_key]
+        if len(nodes) < original_count:
+            logger.info(f"Removed {original_count - len(nodes)} duplicate profile(s) with key: {profile_key}")
+
+        nodes.append(new_node)
+        self._save_nodes(nodes)
+        logger.info(f"Added profile: {profile_key}={profile_value}")
+        return new_node
+
+    def add_batch_sync(self, profiles: list[dict], ref_memory_id: str = "") -> list[MemoryNode]:
+        """Insert many profiles in one write, deduping by key against existing rows."""
+        if not profiles:
+            return []
+
+        nodes = self._load_nodes()
+        new_nodes = [
+            MemoryNode(
+                memory_type=MemoryType.PERSONAL,
+                memory_target=self.memory_target,
+                when_to_use=p.get("profile_key", ""),
+                content=p.get("profile_value", ""),
+                message_time=p.get("message_time", ""),
+                ref_memory_id=ref_memory_id,
+            )
+            for p in profiles
+        ]
+
+        new_keys = {n.when_to_use for n in new_nodes}
+        original_count = len(nodes)
+        nodes = [n for n in nodes if n.when_to_use not in new_keys]
+        if len(nodes) < original_count:
+            logger.info(f"Removed {original_count - len(nodes)} duplicate profile(s) with matching keys")
+
+        nodes.extend(new_nodes)
+        self._save_nodes(nodes)
+        logger.info(f"Batch added {len(new_nodes)} profiles")
+        return new_nodes
+
+    def update_sync(
+        self,
+        profile_id: str,
+        message_time: str,
+        profile_key: str,
+        profile_value: str,
+    ) -> MemoryNode | None:
+        """Update fields for ``profile_id``; return ``None`` if that id is missing."""
+        nodes = self._load_nodes()
+        target_node = None
+        for node in nodes:
+            if node.memory_id == profile_id:
+                node.when_to_use = profile_key
+                node.content = profile_value
+                node.message_time = message_time
+                target_node = node
+                break
+
+        if target_node is None:
+            logger.warning(f"Profile {profile_id} not found")
+            return None
+
+        self._save_nodes(nodes, apply_limits=False)
+        logger.info(f"Updated profile {profile_id}: {profile_key}={profile_value}")
+        return target_node
+
+    def search_sync(self, query: str | list[str], limit: int = 5) -> list[MemoryNode]:
+        """Simple substring/token match over key and content, best matches first."""
+        queries = [query] if isinstance(query, str) else query
+        query_terms = [q.strip().lower() for q in queries if q and q.strip()]
+        if not query_terms:
+            return []
+
+        scored_nodes = []
+        for node in self.get_all_sync():
+            profile_key = str(node.metadata.get("profile_key", node.when_to_use)).lower()
+            haystack = f"{profile_key}: {node.content}".lower()
+            score = 0
+            for term in query_terms:
+                if term in haystack:
+                    score += len(term) + 10
+                else:
+                    token_hits = sum(1 for token in term.split() if token and token in haystack)
+                    score += token_hits
+
+            if score > 0:
+                node.score = float(score)
+                scored_nodes.append(node)
+
+        scored_nodes.sort(key=lambda n: (n.score, n.message_time), reverse=True)
+        return scored_nodes[:limit]
+
+    async def get_all(self) -> list[MemoryNode]:
+        return self.get_all_sync()
+
+    async def get_by(self, *, profile_id: str | None = None, profile_key: str | None = None) -> MemoryNode | None:
+        return self.get_by_sync(profile_id=profile_id, profile_key=profile_key)
+
+    async def delete(self, profile_id: str | list[str]) -> bool | int:
+        return self.delete_sync(profile_id)
+
+    async def delete_all(self) -> int:
+        return self.delete_all_sync()
+
+    async def add(
+        self,
+        message_time: str,
+        profile_key: str,
+        profile_value: str,
+        ref_memory_id: str = "",
+    ) -> MemoryNode:
+        return self.add_sync(message_time, profile_key, profile_value, ref_memory_id)
+
+    async def add_batch(self, profiles: list[dict], ref_memory_id: str = "") -> list[MemoryNode]:
+        return self.add_batch_sync(profiles, ref_memory_id)
+
+    async def update(
+        self,
+        profile_id: str,
+        message_time: str,
+        profile_key: str,
+        profile_value: str,
+    ) -> MemoryNode | None:
+        return self.update_sync(profile_id, message_time, profile_key, profile_value)
+
+    async def search(self, query: str | list[str], limit: int = 5) -> list[MemoryNode]:
+        return self.search_sync(query, limit)

reme/memory/vector_tools/profiles/profile_backend.py

@@ -0,0 +1,51 @@
+"""Profile backend abstractions."""
+
+from abc import ABC, abstractmethod
+
+from ....core.schema import MemoryNode
+
+
+class BaseProfileBackend(ABC):
+    """Abstract interface for profile storage backends."""
+
+    def __init__(self, memory_target: str, max_capacity: int = 50):
+        self.memory_target = memory_target
+        self.max_capacity = max_capacity
+
+    @abstractmethod
+    async def get_all(self) -> list[MemoryNode]:
+        """Return all profile rows for the current user."""
+
+    @abstractmethod
+    async def get_by(self, *, profile_id: str | None = None, profile_key: str | None = None) -> MemoryNode | None:
+        """Return one profile row by id or key."""
+
+    @abstractmethod
+    async def delete(self, profile_id: str | list[str]) -> bool | int:
+        """Delete one or more profile rows."""
+
+    @abstractmethod
+    async def delete_all(self) -> int:
+        """Delete all profile rows for the current user."""
+
+    @abstractmethod
+    async def add(self, message_time: str, profile_key: str, profile_value: str, ref_memory_id: str = "") -> MemoryNode:
+        """Add a single profile row."""
+
+    @abstractmethod
+    async def add_batch(self, profiles: list[dict], ref_memory_id: str = "") -> list[MemoryNode]:
+        """Add multiple profile rows."""
+
+    @abstractmethod
+    async def update(
+        self,
+        profile_id: str,
+        message_time: str,
+        profile_key: str,
+        profile_value: str,
+    ) -> MemoryNode | None:
+        """Update one profile row."""
+
+    @abstractmethod
+    async def search(self, query: str | list[str], limit: int = 5) -> list[MemoryNode]:
+        """Search profile rows relevant to the query."""

reme/memory/vector_tools/profiles/profile_handler.py

@@ -1,195 +1,124 @@
-"""Profile Handler for managing user profiles in local memory"""
+"""Profile handler facade for filesystem and vector backends."""
 
+# pylint: disable=missing-function-docstring
+
+import asyncio
 from pathlib import Path
 
 from loguru import logger
 
-from ....core.enumeration import MemoryType
+from .file_profile_backend import FileProfileBackend
+from .profile_backend import BaseProfileBackend
+from .vector_profile_backend import VectorProfileBackend
+from ....core import ServiceContext
 from ....core.schema import MemoryNode
-from ....core.utils import CacheHandler, deduplicate_memories
 
 
 class ProfileHandler:
-    """User profile CRUD handler"""
+    """User profile facade with pluggable storage backends."""
 
-    def __init__(self, profile_path: str | Path, memory_target: str, max_capacity: int = 50):
-        """init"""
-        self.memory_target: str = memory_target
-        self.cache_key: str = self.memory_target.replace(" ", "_").lower()
-        self.cache_handler: CacheHandler = CacheHandler(profile_path)
-        self.max_capacity: int = max_capacity
-
-    def _load_nodes(self) -> list[MemoryNode]:
-        """Load profile nodes"""
-        cached_data = self.cache_handler.load(self.cache_key, auto_clean=False)
-        if not cached_data:
-            return []
-        return [MemoryNode(**data) for data in cached_data]
-
-    def _save_nodes(self, nodes: list[MemoryNode], apply_limits: bool = True):
-        """Save nodes with optional deduplication and capacity enforcement"""
-        if apply_limits:
-            nodes = deduplicate_memories(nodes)
-
-            # Enforce capacity limit by removing the oldest profiles
-            if len(nodes) > self.max_capacity:
-                sorted_nodes = sorted(nodes, key=lambda n: n.message_time)
-                removed_count = len(sorted_nodes) - self.max_capacity
-                nodes = sorted_nodes[removed_count:]
-                logger.info(
-                    f"Capacity limit reached: removed {removed_count} oldest profiles "
-                    f"(kept {len(nodes)}/{self.max_capacity})",
-                )
-
-        nodes_data = [node.model_dump(exclude_none=True) for node in nodes]
-        self.cache_handler.save(self.cache_key, nodes_data)
-        logger.info(f"Saved {len(nodes)} profiles to {self.cache_key}")
-
-    def delete(self, profile_id: str | list[str]) -> bool | int:
-        """Delete profile by ID(s), returns True/False for single ID or count for batch delete"""
-        nodes = self._load_nodes()
-        original_count = len(nodes)
-
-        # Batch delete mode
-        if isinstance(profile_id, list):
-            profile_ids_set = set(profile_id)
-            nodes = [n for n in nodes if n.memory_id not in profile_ids_set]
-            deleted_count = original_count - len(nodes)
-
-            if deleted_count == 0:
-                logger.warning(f"No profiles found to delete from {len(profile_id)} IDs")
-                return 0
-
-            self._save_nodes(nodes, apply_limits=False)
-            logger.info(f"Batch deleted {deleted_count} profiles")
-            return deleted_count
-
-        # Single delete mode
-        nodes = [n for n in nodes if n.memory_id != profile_id]
-
-        if len(nodes) == original_count:
-            logger.warning(f"Profile {profile_id} not found")
-            return False
-
-        self._save_nodes(nodes, apply_limits=False)
-        logger.info(f"Deleted profile {profile_id}")
-        return True
-
-    def delete_all(self) -> int:
-        """Delete all profiles, returns count deleted"""
-        nodes = self._load_nodes()
-        count = len(nodes)
-        self._save_nodes([], apply_limits=False)
-        logger.info(f"Deleted all {count} profiles")
-        return count
-
-    def add(self, message_time: str, profile_key: str, profile_value: str, ref_memory_id: str = "") -> MemoryNode:
-        """Add new profile, returns created MemoryNode"""
-        nodes = self._load_nodes()
-
-        new_node = MemoryNode(
-            memory_type=MemoryType.PERSONAL,
-            memory_target=self.memory_target,
-            when_to_use=profile_key,
-            content=profile_value,
-            message_time=message_time,
-            ref_memory_id=ref_memory_id,
+    def __init__(
+        self,
+        memory_target: str,
+        profile_path: str | Path | None = None,
+        service_context: ServiceContext | None = None,
+        profile_backend: str = "filesystem",
+        profile_store_name: str = "profile",
+        max_capacity: int = 50,
+    ):
+        self.memory_target = memory_target
+        self.profile_backend = profile_backend
+        self.profile_store_name = profile_store_name
+        self.max_capacity = max_capacity
+        self.cache_key = self.memory_target.replace(" ", "_").lower()
+        self.backend = self._build_backend(
+            profile_path=profile_path,
+            service_context=service_context,
         )
 
-        # Remove existing nodes with the same when_to_use (profile_key)
-        original_count = len(nodes)
-        nodes = [n for n in nodes if n.when_to_use != profile_key]
-        if len(nodes) < original_count:
-            logger.info(f"Removed {original_count - len(nodes)} duplicate profile(s) with key: {profile_key}")
-
-        nodes.append(new_node)
-        self._save_nodes(nodes)
-        logger.info(f"Added profile: {profile_key}={profile_value}")
-        return new_node
-
-    def add_batch(self, profiles: list[dict], ref_memory_id: str = "") -> list[MemoryNode]:
-        """Add multiple profiles in batch, returns list of created MemoryNodes"""
-        if not profiles:
-            return []
-
-        nodes = self._load_nodes()
-
-        new_nodes = [
-            MemoryNode(
-                memory_type=MemoryType.PERSONAL,
+    def _build_backend(
+        self,
+        profile_path: str | Path | None,
+        service_context: ServiceContext | None,
+    ) -> BaseProfileBackend:
+        if self.profile_backend == "filesystem":
+            if profile_path is None:
+                raise ValueError("profile_path is required for filesystem profile backend")
+            return FileProfileBackend(
+                profile_path=profile_path,
                 memory_target=self.memory_target,
-                when_to_use=p.get("profile_key", ""),
-                content=p.get("profile_value", ""),
-                message_time=p.get("message_time", ""),
-                ref_memory_id=ref_memory_id,
+                max_capacity=self.max_capacity,
             )
-            for p in profiles
-        ]
 
-        # Remove existing nodes with the same when_to_use (profile_key)
-        new_keys = {n.when_to_use for n in new_nodes}
-        original_count = len(nodes)
-        nodes = [n for n in nodes if n.when_to_use not in new_keys]
-        if len(nodes) < original_count:
-            logger.info(f"Removed {original_count - len(nodes)} duplicate profile(s) with matching keys")
+        if self.profile_backend == "vector":
+            if service_context is None:
+                raise ValueError("service_context is required for vector profile backend")
+            return VectorProfileBackend(
+                memory_target=self.memory_target,
+                service_context=service_context,
+                vector_store_name=self.profile_store_name,
+                max_capacity=self.max_capacity,
+            )
 
-        nodes.extend(new_nodes)
-        self._save_nodes(nodes)
-        logger.info(f"Batch added {len(new_nodes)} profiles")
-        return new_nodes
-
-    def update(self, profile_id: str, message_time: str, profile_key: str, profile_value: str) -> MemoryNode | None:
-        """Update profile by ID, returns updated node or None if not found"""
-        nodes = self._load_nodes()
-
-        target_node = None
-        for node in nodes:
-            if node.memory_id == profile_id:
-                node.when_to_use = profile_key
-                node.content = profile_value
-                node.message_time = message_time
-                target_node = node
-                break
-
-        if target_node is None:
-            logger.warning(f"Profile {profile_id} not found")
-            return None
-
-        self._save_nodes(nodes, apply_limits=False)
-        logger.info(f"Updated profile {profile_id}: {profile_key}={profile_value}")
-        return target_node
-
-    def get_by(self, *, profile_id: str | None = None, profile_key: str | None = None) -> MemoryNode | None:
-        """Get profile by ID or key"""
-        if not profile_id and not profile_key:
-            raise ValueError("Must provide either profile_id or profile_key")
-
-        nodes = self._load_nodes()
-        for node in nodes:
-            if profile_id and node.memory_id == profile_id:
-                return node
-            if profile_key and node.when_to_use == profile_key:
-                return node
-        return None
-
-    def get_by_id(self, profile_id: str) -> MemoryNode | None:
-        """Get profile by ID (convenience method)"""
-        return self.get_by(profile_id=profile_id)
-
-    def get_by_key(self, profile_key: str) -> MemoryNode | None:
-        """Get profile by key (convenience method)"""
-        return self.get_by(profile_key=profile_key)
-
-    def get_all(self) -> list[MemoryNode]:
-        """Get all profiles, sorted by message_time"""
-        nodes = self._load_nodes()
-        nodes.sort(key=lambda n: n.message_time)
-        return nodes
+        raise ValueError(f"Unsupported profile backend: {self.profile_backend}")
 
     @staticmethod
-    def _format_node(node: MemoryNode, add_profile_id: bool = False, add_history_id: bool = False) -> str:
-        """Format a single node to string"""
+    def _run_sync(coro):
+        try:
+            asyncio.get_running_loop()
+        except RuntimeError:
+            return asyncio.run(coro)
+        raise RuntimeError(
+            "Synchronous profile access is not available in an active event loop. Use async methods instead.",
+        )
+
+    async def adelete(self, profile_id: str | list[str]) -> bool | int:
+        return await self.backend.delete(profile_id)
+
+    async def adelete_all(self) -> int:
+        return await self.backend.delete_all()
+
+    async def aadd(
+        self,
+        message_time: str,
+        profile_key: str,
+        profile_value: str,
+        ref_memory_id: str = "",
+    ) -> MemoryNode:
+        return await self.backend.add(message_time, profile_key, profile_value, ref_memory_id)
+
+    async def aadd_batch(self, profiles: list[dict], ref_memory_id: str = "") -> list[MemoryNode]:
+        return await self.backend.add_batch(profiles, ref_memory_id)
+
+    async def aupdate(
+        self,
+        profile_id: str,
+        message_time: str,
+        profile_key: str,
+        profile_value: str,
+    ) -> MemoryNode | None:
+        return await self.backend.update(profile_id, message_time, profile_key, profile_value)
+
+    async def aget_by(self, *, profile_id: str | None = None, profile_key: str | None = None) -> MemoryNode | None:
+        return await self.backend.get_by(profile_id=profile_id, profile_key=profile_key)
+
+    async def aget_by_id(self, profile_id: str) -> MemoryNode | None:
+        return await self.aget_by(profile_id=profile_id)
+
+    async def aget_by_key(self, profile_key: str) -> MemoryNode | None:
+        return await self.aget_by(profile_key=profile_key)
+
+    async def aget_all(self) -> list[MemoryNode]:
+        return await self.backend.get_all()
+
+    async def asearch(self, query: str | list[str], limit: int = 5) -> list[MemoryNode]:
+        return await self.backend.search(query=query, limit=limit)
+
+    @staticmethod
+    def format_node(node: MemoryNode, add_profile_id: bool = False, add_history_id: bool = False) -> str:
+        """Render a profile ``MemoryNode`` as a single-line string for tools/logs."""
         parts = []
+        profile_key = str(node.metadata.get("profile_key", node.when_to_use))
 
         if add_profile_id:
             parts.append(f"profile_id={node.memory_id}")
@@ -197,16 +126,70 @@ class ProfileHandler:
         if node.message_time:
             parts.append(f"[{node.message_time}]")
 
-        parts.append(f"{node.when_to_use}: {node.content}")
+        parts.append(f"{profile_key}: {node.content}")
 
-        if add_history_id:
+        if add_history_id and node.ref_memory_id:
             parts.append(f"history_id={node.ref_memory_id}")
 
         return " ".join(parts)
 
-    def read_all(self, add_profile_id: bool = False, add_history_id: bool = False) -> str:
-        """Read all profiles and return formatted string"""
-        nodes = self.get_all()
-        formatted_profiles = [self._format_node(node, add_profile_id, add_history_id) for node in nodes]
+    async def aread_all(self, add_profile_id: bool = False, add_history_id: bool = False) -> str:
+        nodes = await self.aget_all()
+        formatted_profiles = [self.format_node(node, add_profile_id, add_history_id) for node in nodes]
         logger.info(f"Read {len(formatted_profiles)} profiles from {self.cache_key}")
         return "\n".join(formatted_profiles).strip()
+
+    async def aretrieve(
+        self,
+        query: str | list[str],
+        limit: int = 5,
+        add_profile_id: bool = True,
+        add_history_id: bool = False,
+    ) -> tuple[list[MemoryNode], str]:
+        nodes = await self.asearch(query=query, limit=limit)
+        formatted_profiles = [self.format_node(node, add_profile_id, add_history_id) for node in nodes]
+        return nodes, "\n".join(formatted_profiles).strip()
+
+    def delete(self, profile_id: str | list[str]) -> bool | int:
+        if isinstance(self.backend, FileProfileBackend):
+            return self.backend.delete_sync(profile_id)
+        return self._run_sync(self.adelete(profile_id))
+
+    def delete_all(self) -> int:
+        if isinstance(self.backend, FileProfileBackend):
+            return self.backend.delete_all_sync()
+        return self._run_sync(self.adelete_all())
+
+    def add(self, message_time: str, profile_key: str, profile_value: str, ref_memory_id: str = "") -> MemoryNode:
+        if isinstance(self.backend, FileProfileBackend):
+            return self.backend.add_sync(message_time, profile_key, profile_value, ref_memory_id)
+        return self._run_sync(self.aadd(message_time, profile_key, profile_value, ref_memory_id))
+
+    def add_batch(self, profiles: list[dict], ref_memory_id: str = "") -> list[MemoryNode]:
+        if isinstance(self.backend, FileProfileBackend):
+            return self.backend.add_batch_sync(profiles, ref_memory_id)
+        return self._run_sync(self.aadd_batch(profiles, ref_memory_id))
+
+    def update(self, profile_id: str, message_time: str, profile_key: str, profile_value: str) -> MemoryNode | None:
+        if isinstance(self.backend, FileProfileBackend):
+            return self.backend.update_sync(profile_id, message_time, profile_key, profile_value)
+        return self._run_sync(self.aupdate(profile_id, message_time, profile_key, profile_value))
+
+    def get_by(self, *, profile_id: str | None = None, profile_key: str | None = None) -> MemoryNode | None:
+        if isinstance(self.backend, FileProfileBackend):
+            return self.backend.get_by_sync(profile_id=profile_id, profile_key=profile_key)
+        return self._run_sync(self.aget_by(profile_id=profile_id, profile_key=profile_key))
+
+    def get_by_id(self, profile_id: str) -> MemoryNode | None:
+        return self._run_sync(self.aget_by_id(profile_id))
+
+    def get_by_key(self, profile_key: str) -> MemoryNode | None:
+        return self._run_sync(self.aget_by_key(profile_key))
+
+    def get_all(self) -> list[MemoryNode]:
+        if isinstance(self.backend, FileProfileBackend):
+            return self.backend.get_all_sync()
+        return self._run_sync(self.aget_all())
+
+    def read_all(self, add_profile_id: bool = False, add_history_id: bool = False) -> str:
+        return self._run_sync(self.aread_all(add_profile_id, add_history_id))

reme/memory/vector_tools/profiles/profile_vector_handler.py

@@ -0,0 +1,245 @@
+"""Vector-backed handler for bounded user profiles."""
+
+import hashlib
+
+from loguru import logger
+
+from ....core import ServiceContext
+from ....core.enumeration import MemoryType
+from ....core.schema import MemoryNode
+from ....core.vector_store import BaseVectorStore
+
+
+class ProfileVectorHandler:
+    """Manage profile rows stored in a dedicated vector collection."""
+
+    PROFILE_KIND = "profile"
+
+    def __init__(
+        self,
+        memory_target: str,
+        service_context: ServiceContext,
+        vector_store_name: str = "profile",
+        max_capacity: int = 50,
+    ):
+        self.memory_target = memory_target
+        self.service_context = service_context
+        self.vector_store_name = vector_store_name
+        self.max_capacity = max_capacity
+        self.vector_store: BaseVectorStore = service_context.vector_stores[vector_store_name]
+
+    @staticmethod
+    def build_retrieval_text(profile_key: str, profile_value: str) -> str:
+        """Build the text that will be embedded for semantic profile retrieval."""
+        return f"{profile_key}: {profile_value}".strip(": ")
+
+    def build_profile_id(self, profile_key: str) -> str:
+        """Build a stable id from user and key."""
+        hash_obj = hashlib.sha256(f"{self.memory_target}\n{profile_key}".encode("utf-8"))
+        return hash_obj.hexdigest()[:16]
+
+    def _base_filters(self) -> dict:
+        """Filters shared by all profile rows in the vector collection."""
+        return {
+            "memory_type": MemoryType.IDENTITY.value,
+            "memory_target": self.memory_target,
+            "profile_kind": self.PROFILE_KIND,
+        }
+
+    def _build_profile_node(self, profile: dict, ref_memory_id: str = "") -> MemoryNode:
+        """Turn a profile dict into a ``MemoryNode`` for upsert into the vector store."""
+        profile_key = profile.get("profile_key", "").strip()
+        profile_value = profile.get("profile_value", "").strip()
+        message_time = profile.get("message_time", "")
+        ref_id = profile.get("ref_memory_id", ref_memory_id)
+        metadata = dict(profile.get("metadata", {}))
+        metadata.update(
+            {
+                "profile_key": profile_key,
+                "profile_kind": self.PROFILE_KIND,
+                "profile_backend": "vector",
+            },
+        )
+        return MemoryNode(
+            memory_id=self.build_profile_id(profile_key),
+            memory_type=MemoryType.IDENTITY,
+            memory_target=self.memory_target,
+            when_to_use=self.build_retrieval_text(profile_key, profile_value),
+            content=profile_value,
+            message_time=message_time,
+            ref_memory_id=ref_id,
+            metadata=metadata,
+        )
+
+    def _vector_profile_matches(self, memory_node: MemoryNode) -> bool:
+        """True if ``memory_node`` belongs to this handler's target and profile kind."""
+        if memory_node.memory_target != self.memory_target:
+            return False
+        if memory_node.memory_type is not MemoryType.IDENTITY:
+            return False
+        if memory_node.metadata.get("profile_kind") != self.PROFILE_KIND:
+            return False
+        return True
+
+    async def _get_by_profile_id(self, profile_id: str) -> MemoryNode | None:
+        """Load by vector id and validate filters."""
+        try:
+            vector_node = await self.vector_store.get(profile_id)
+        except KeyError:
+            logger.warning(f"Profile {profile_id} not found in vector store")
+            return None
+        if vector_node is None:
+            logger.warning(f"Profile {profile_id} not found in vector store")
+            return None
+        memory_node = MemoryNode.from_vector_node(vector_node)
+        if not self._vector_profile_matches(memory_node):
+            return None
+        return memory_node
+
+    async def _get_by_profile_key(self, profile_key: str) -> MemoryNode | None:
+        """Load the single row matching ``profile_key`` under base filters."""
+        filters = {**self._base_filters(), "profile_key": profile_key}
+        vector_nodes = await self.vector_store.list(filters=filters, limit=1)
+        if not vector_nodes:
+            return None
+        return MemoryNode.from_vector_node(vector_nodes[0])
+
+    async def get_all(self) -> list[MemoryNode]:
+        """List every profile row for this memory target, sorted by store."""
+        vector_nodes = await self.vector_store.list(
+            filters=self._base_filters(),
+            sort_key="message_time",
+            reverse=False,
+        )
+        return [MemoryNode.from_vector_node(node) for node in vector_nodes]
+
+    async def get_by(self, *, profile_id: str | None = None, profile_key: str | None = None) -> MemoryNode | None:
+        """Return one profile by stable id or by logical profile key."""
+        if not profile_id and not profile_key:
+            raise ValueError("Must provide either profile_id or profile_key")
+        if profile_id:
+            return await self._get_by_profile_id(profile_id)
+        return await self._get_by_profile_key(profile_key or "")
+
+    async def delete(self, profile_id: str | list[str]) -> bool | int:
+        """Delete one id, many ids, or report zero/false when nothing matched."""
+        if isinstance(profile_id, list):
+            profile_ids = list(dict.fromkeys(pid for pid in profile_id if pid))
+            if not profile_ids:
+                return 0
+            existing_nodes = []
+            for pid in profile_ids:
+                node = await self.get_by(profile_id=pid)
+                if node is not None:
+                    existing_nodes.append(node)
+            if not existing_nodes:
+                return 0
+            await self.vector_store.delete([node.memory_id for node in existing_nodes])
+            return len(existing_nodes)
+
+        existing_node = await self.get_by(profile_id=profile_id)
+        if existing_node is None:
+            return False
+        await self.vector_store.delete(existing_node.memory_id)
+        return True
+
+    async def delete_all(self) -> int:
+        """Remove all profile vectors for this target; returns how many were deleted."""
+        nodes = await self.get_all()
+        if not nodes:
+            return 0
+        await self.vector_store.delete([node.memory_id for node in nodes])
+        return len(nodes)
+
+    async def add_batch(self, profiles: list[dict], ref_memory_id: str = "") -> list[MemoryNode]:
+        """Upsert many profiles at once (last dict wins per key), then enforce capacity."""
+        if not profiles:
+            return []
+
+        deduped_profiles: dict[str, dict] = {}
+        for profile in profiles:
+            profile_key = profile.get("profile_key", "").strip()
+            if not profile_key:
+                continue
+            deduped_profiles[profile_key] = profile
+
+        new_nodes = [
+            self._build_profile_node(profile, ref_memory_id=ref_memory_id) for profile in deduped_profiles.values()
+        ]
+        if not new_nodes:
+            return []
+
+        await self.vector_store.delete([node.memory_id for node in new_nodes])
+        await self.vector_store.insert([node.to_vector_node() for node in new_nodes])
+        await self.enforce_capacity()
+        return new_nodes
+
+    async def add(self, message_time: str, profile_key: str, profile_value: str, ref_memory_id: str = "") -> MemoryNode:
+        """Insert or replace a single profile row."""
+        nodes = await self.add_batch(
+            [
+                {
+                    "message_time": message_time,
+                    "profile_key": profile_key,
+                    "profile_value": profile_value,
+                },
+            ],
+            ref_memory_id=ref_memory_id,
+        )
+        return nodes[0]
+
+    async def update(
+        self,
+        profile_id: str,
+        message_time: str,
+        profile_key: str,
+        profile_value: str,
+    ) -> MemoryNode | None:
+        """Replace content and key for ``profile_id``; return ``None`` if missing."""
+        existing_node = await self.get_by(profile_id=profile_id)
+        if existing_node is None:
+            return None
+
+        new_node = self._build_profile_node(
+            {
+                "message_time": message_time,
+                "profile_key": profile_key,
+                "profile_value": profile_value,
+                "ref_memory_id": existing_node.ref_memory_id,
+                "metadata": existing_node.metadata,
+            },
+        )
+
+        if existing_node.memory_id != new_node.memory_id:
+            await self.vector_store.delete(existing_node.memory_id)
+        else:
+            await self.vector_store.delete(new_node.memory_id)
+
+        await self.vector_store.insert(new_node.to_vector_node())
+        await self.enforce_capacity()
+        return new_node
+
+    async def search(self, query: str | list[str], limit: int = 5) -> list[MemoryNode]:
+        """Semantic search with de-duplication across multiple query strings."""
+        queries = [query] if isinstance(query, str) else query
+        seen_nodes: dict[str, MemoryNode] = {}
+        for item in queries:
+            if not item or not item.strip():
+                continue
+            vector_nodes = await self.vector_store.search(item, limit=limit, filters=self._base_filters())
+            for vector_node in vector_nodes:
+                memory_node = MemoryNode.from_vector_node(vector_node)
+                seen_nodes[memory_node.memory_id] = memory_node
+        nodes = list(seen_nodes.values())
+        nodes.sort(key=lambda node: (node.score, node.message_time), reverse=True)
+        return nodes[:limit]
+
+    async def enforce_capacity(self):
+        """Drop oldest rows when count exceeds ``max_capacity``."""
+        nodes = await self.get_all()
+        overflow = len(nodes) - self.max_capacity
+        if overflow <= 0:
+            return
+
+        to_delete = [node.memory_id for node in nodes[:overflow]]
+        await self.vector_store.delete(to_delete)

reme/memory/vector_tools/profiles/read_all_profiles.py

@@ -1,8 +1,7 @@
-"""Read user profile tool"""
+"""Read user profile tool."""
 
 from loguru import logger
 
-from .profile_handler import ProfileHandler
 from ..base_memory_tool import BaseMemoryTool
 from ....core.schema import ToolCall
 
@@ -44,8 +43,8 @@ class ReadAllProfiles(BaseMemoryTool):
         else:
             target = self.memory_target
 
-        profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=target)
-        profiles_str = profile_handler.read_all(add_profile_id=True)
+        profile_handler = self.get_profile_handler(target)
+        profiles_str = await profile_handler.aread_all(add_profile_id=True)
         if not profiles_str:
             output = "No profiles found."
             logger.info(output)

reme/memory/vector_tools/profiles/retrieve_profile.py

@@ -0,0 +1,102 @@
+"""Retrieve relevant profile rows."""
+
+from loguru import logger
+
+from .profile_handler import ProfileHandler
+from ..base_memory_tool import BaseMemoryTool
+from ....core.schema import MemoryNode, ToolCall
+
+
+class RetrieveProfile(BaseMemoryTool):
+    """Tool to retrieve relevant profiles using the configured backend."""
+
+    def __init__(self, top_k: int = 5, enable_memory_target: bool = False, **kwargs):
+        super().__init__(**kwargs)
+        self.top_k = top_k
+        self.enable_memory_target = enable_memory_target
+
+    def _build_query_parameters(self) -> dict:
+        properties = {
+            "query": {
+                "type": "string",
+                "description": "query",
+            },
+        }
+        required = ["query"]
+        if self.enable_memory_target:
+            properties["memory_target"] = {
+                "type": "string",
+                "description": "memory_target",
+            }
+            required.append("memory_target")
+        return {
+            "type": "object",
+            "properties": properties,
+            "required": required,
+        }
+
+    def _build_tool_call(self) -> ToolCall:
+        return ToolCall(
+            **{
+                "description": "Retrieve relevant user profiles using semantic matching.",
+                "parameters": self._build_query_parameters(),
+            },
+        )
+
+    def _build_multiple_tool_call(self) -> ToolCall:
+        return ToolCall(
+            **{
+                "description": "Retrieve relevant user profiles using semantic matching.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "query_items": {
+                            "type": "array",
+                            "description": "List of query items.",
+                            "items": self._build_query_parameters(),
+                        },
+                    },
+                    "required": ["query_items"],
+                },
+            },
+        )
+
+    async def execute(self):
+        if self.enable_multiple:
+            query_items = self.context.get("query_items", [])
+        else:
+            query_items = [self.context]
+
+        queries_by_target: dict[str, list[str]] = {}
+        for item in query_items:
+            target = item["memory_target"] if self.enable_memory_target else self.memory_target
+            queries_by_target.setdefault(target, []).append(item["query"])
+
+        profile_nodes: list[MemoryNode] = []
+        for target, queries in queries_by_target.items():
+            profile_handler = self.get_profile_handler(target)
+            nodes, _ = await profile_handler.aretrieve(
+                query=queries,
+                limit=self.top_k,
+                add_profile_id=True,
+                add_history_id=True,
+            )
+            profile_nodes.extend(nodes)
+
+        seen_ids = {node.memory_id: node for node in self.retrieved_nodes if node.memory_id}
+        new_nodes = []
+        for node in profile_nodes:
+            if node.memory_id not in seen_ids:
+                seen_ids[node.memory_id] = node
+                new_nodes.append(node)
+        self.retrieved_nodes.extend(new_nodes)
+
+        if not new_nodes:
+            output = "No new profiles found."
+        else:
+            output = "\n".join(
+                [ProfileHandler.format_node(node, add_profile_id=True, add_history_id=True) for node in new_nodes],
+            )
+
+        logger.info(f"Retrieved {len(profile_nodes)} profiles, {len(new_nodes)} new after deduplication")
+        return output

reme/memory/vector_tools/profiles/update_profile.py

@@ -1,8 +1,7 @@
-"""Update user profile tool"""
+"""Update user profile tool."""
 
 from loguru import logger
 
-from .profile_handler import ProfileHandler
 from ..base_memory_tool import BaseMemoryTool
 from ....core.schema import ToolCall
 
@@ -82,8 +81,8 @@ class UpdateProfile(BaseMemoryTool):
 
         # Delete profiles (using self.memory_target)
         if profile_ids_to_delete:
-            profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=self.memory_target)
-            removed_count = profile_handler.delete(profile_ids_to_delete)
+            profile_handler = self.get_profile_handler(self.memory_target)
+            removed_count = await profile_handler.adelete(profile_ids_to_delete)
 
         # Add new profiles
         if profiles_to_add:
@@ -98,14 +97,17 @@ class UpdateProfile(BaseMemoryTool):
 
                 # Add profiles for each target
                 for target, target_profiles in profiles_by_target.items():
-                    profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=target)
-                    new_nodes = profile_handler.add_batch(profiles=target_profiles, ref_memory_id=self.history_id)
+                    profile_handler = self.get_profile_handler(target)
+                    new_nodes = await profile_handler.aadd_batch(
+                        profiles=target_profiles,
+                        ref_memory_id=self.history_id,
+                    )
                     self.memory_nodes.extend(new_nodes)
                     added_count += len(new_nodes)
             else:
                 # Use self.memory_target for all profiles
-                profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=self.memory_target)
-                new_nodes = profile_handler.add_batch(profiles=profiles_to_add, ref_memory_id=self.history_id)
+                profile_handler = self.get_profile_handler(self.memory_target)
+                new_nodes = await profile_handler.aadd_batch(profiles=profiles_to_add, ref_memory_id=self.history_id)
                 self.memory_nodes.extend(new_nodes)
                 added_count = len(new_nodes)
 

reme/memory/vector_tools/profiles/update_profiles_v1.py

@@ -1,8 +1,7 @@
-"""Update user profile tool"""
+"""Update user profile tool."""
 
 from loguru import logger
 
-from .profile_handler import ProfileHandler
 from ..base_memory_tool import BaseMemoryTool
 from ....core.schema import ToolCall
 
@@ -113,8 +112,8 @@ class UpdateProfilesV1(BaseMemoryTool):
             for target, profile_ids in delete_by_target.items():
                 if profile_ids:
                     profile_ids = sorted(set(profile_ids))  # Remove duplicates and sort
-                    profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=target)
-                    profile_handler.delete(profile_ids)
+                    profile_handler = self.get_profile_handler(target)
+                    await profile_handler.adelete(profile_ids)
 
         # Step 2: Prepare all profiles to add (both updated and new)
         all_profiles_to_add = []
@@ -158,8 +157,8 @@ class UpdateProfilesV1(BaseMemoryTool):
         added_count = len(profiles_to_add)
 
         for target, target_profiles in profiles_by_target.items():
-            profile_handler = ProfileHandler(profile_path=self.profile_path, memory_target=target)
-            new_nodes = profile_handler.add_batch(profiles=target_profiles, ref_memory_id=self.history_id)
+            profile_handler = self.get_profile_handler(target)
+            new_nodes = await profile_handler.aadd_batch(profiles=target_profiles, ref_memory_id=self.history_id)
             all_memory_nodes.extend(new_nodes)
 
         # Extend memory_nodes for tracking

reme/memory/vector_tools/profiles/vector_profile_backend.py

@@ -0,0 +1,55 @@
+"""Vector-backed profile storage."""
+
+from .profile_backend import BaseProfileBackend
+from .profile_vector_handler import ProfileVectorHandler
+from ....core import ServiceContext
+from ....core.schema import MemoryNode
+
+
+class VectorProfileBackend(BaseProfileBackend):
+    """Persist user profiles in a dedicated vector store."""
+
+    def __init__(
+        self,
+        memory_target: str,
+        service_context: ServiceContext,
+        vector_store_name: str = "profile",
+        max_capacity: int = 50,
+    ):
+        super().__init__(memory_target=memory_target, max_capacity=max_capacity)
+        self.handler = ProfileVectorHandler(
+            memory_target=memory_target,
+            service_context=service_context,
+            vector_store_name=vector_store_name,
+            max_capacity=max_capacity,
+        )
+
+    async def get_all(self) -> list[MemoryNode]:
+        return await self.handler.get_all()
+
+    async def get_by(self, *, profile_id: str | None = None, profile_key: str | None = None) -> MemoryNode | None:
+        return await self.handler.get_by(profile_id=profile_id, profile_key=profile_key)
+
+    async def delete(self, profile_id: str | list[str]) -> bool | int:
+        return await self.handler.delete(profile_id)
+
+    async def delete_all(self) -> int:
+        return await self.handler.delete_all()
+
+    async def add(self, message_time: str, profile_key: str, profile_value: str, ref_memory_id: str = "") -> MemoryNode:
+        return await self.handler.add(message_time, profile_key, profile_value, ref_memory_id)
+
+    async def add_batch(self, profiles: list[dict], ref_memory_id: str = "") -> list[MemoryNode]:
+        return await self.handler.add_batch(profiles, ref_memory_id)
+
+    async def update(
+        self,
+        profile_id: str,
+        message_time: str,
+        profile_key: str,
+        profile_value: str,
+    ) -> MemoryNode | None:
+        return await self.handler.update(profile_id, message_time, profile_key, profile_value)
+
+    async def search(self, query: str | list[str], limit: int = 5) -> list[MemoryNode]:
+        return await self.handler.search(query, limit)

reme/reme.py

@@ -14,6 +14,7 @@ from .memory.vector_tools import (
     DelegateTask,
     ReadAllProfiles,
     ReadHistory,
+    RetrieveProfile,
     RetrieveMemory,
     UpdateProfilesV1,
 )
@@ -46,6 +47,7 @@ class ReMe(Application):
         config_path: str = "vector",
         enable_logo: bool = True,
         log_to_console: bool = True,
+        log_to_file: bool = True,
         default_llm_config: dict | None = None,
         default_embedding_model_config: dict | None = None,
         default_vector_store_config: dict | None = None,
@@ -54,6 +56,10 @@ class ReMe(Application):
         target_task_names: list[str] | None = None,
         target_tool_names: list[str] | None = None,
         enable_profile: bool = True,
+        profile_backend: str = "filesystem",
+        profile_store_name: str = "profile",
+        profile_collection_name: str | None = None,
+        profile_max_capacity: int = 50,
         **kwargs,
     ):
         """Initialize ReMe with config.
@@ -68,8 +74,37 @@ class ReMe(Application):
             ```
 
         Args:
+            *args: Positional arguments forwarded to the base `Application`.
+            llm_api_key: API key used by the default LLM backend when provided.
+            llm_base_url: Base URL used by the default LLM backend when provided.
+            embedding_api_key: API key used by the default embedding backend when provided.
+            embedding_base_url: Base URL used by the default embedding backend when provided.
+            working_dir: Directory for generated config, logs, caches, and local stores.
+            config_path: Built-in config name or config file path used to initialize services.
+            enable_logo: Whether to print the ReMe logo during startup.
+            log_to_console: Whether to emit logs to the console.
+            log_to_file: Whether to write logs under `working_dir`.
+            default_llm_config: Overrides for the default LLM configuration.
+            default_embedding_model_config: Overrides for the default embedding model configuration.
+            default_vector_store_config: Configuration for the default memory vector store.
+                Its `collection_name` is used for normal memory storage.
+            default_token_counter_config: Overrides for the default token counter configuration.
+            target_user_names: Personal memory targets to register at initialization.
+            target_task_names: Procedural memory targets to register at initialization.
+            target_tool_names: Tool memory targets to register at initialization.
             enable_profile: Whether to enable profile functionality. Set to False when using
-                cloud-based vector stores to avoid local file operations. Default is True.
+                profile-free memory flows.
+            profile_backend: Profile storage backend. Use "filesystem" for local JSONL profile
+                files or "vector" for a dedicated profile vector collection.
+            profile_store_name: Internal vector store key used to register and look up the
+                profile vector store in `service_context.vector_stores`. This is not the
+                database collection name.
+            profile_collection_name: Dedicated database collection/table name for vector
+                profiles. When unset, vector profiles use the default memory collection name
+                with a "_profile" suffix.
+            profile_max_capacity: Maximum number of profile rows to keep per memory target.
+                When the limit is exceeded, the oldest profile rows are removed.
+            **kwargs: Additional keyword arguments forwarded to the base `Application`.
         """
         super().__init__(
             *args,
@@ -81,6 +116,7 @@ class ReMe(Application):
             config_path=config_path,
             enable_logo=enable_logo,
             log_to_console=log_to_console,
+            log_to_file=log_to_file,
             parser=ReMeConfigParser,
             default_llm_config=default_llm_config,
             default_embedding_model_config=default_embedding_model_config,
@@ -90,6 +126,10 @@ class ReMe(Application):
         )
 
         self.enable_profile = enable_profile
+        self.profile_backend = profile_backend
+        self.profile_store_name = profile_store_name
+        self.profile_collection_name = profile_collection_name
+        self.profile_max_capacity = profile_max_capacity
 
         memory_target_type_mapping: dict[str, MemoryType] = {}
         if target_user_names:
@@ -109,13 +149,16 @@ class ReMe(Application):
 
         self.service_context.memory_target_type_mapping = memory_target_type_mapping
 
-        if self.enable_profile:
+        if self.enable_profile and self.profile_backend == "filesystem":
             profile_path = Path(self.service_context.service_config.working_dir) / "profile"
             profile_path.mkdir(parents=True, exist_ok=True)
             self.profile_dir: str = str(profile_path)
         else:
             self.profile_dir: str = ""
 
+        if self.enable_profile and self.profile_backend == "vector":
+            self._ensure_profile_vector_store_config()
+
     def _add_meta_memory(self, memory_type: str | MemoryType, memory_target: str):
         """Register or validate a memory target with the given memory type."""
         if memory_target in self.service_context.memory_target_type_mapping:
@@ -184,6 +227,38 @@ class ReMe(Application):
             return result
         return result["answer"]
 
+    def _ensure_profile_vector_store_config(self) -> None:
+        """Ensure the dedicated profile vector store exists in service config."""
+        vector_store_configs = self.service_context.service_config.vector_stores
+        if "default" not in vector_store_configs:
+            raise RuntimeError("Vector profile backend requires a default vector store configuration")
+
+        default_config = vector_store_configs["default"]
+        profile_collection_name = self.profile_collection_name or f"{default_config.collection_name}_profile"
+
+        if self.profile_store_name in vector_store_configs:
+            if self.profile_collection_name:
+                vector_store_configs[self.profile_store_name] = vector_store_configs[
+                    self.profile_store_name
+                ].model_copy(
+                    update={"collection_name": profile_collection_name},
+                )
+            return
+
+        vector_store_configs[self.profile_store_name] = default_config.model_copy(
+            update={"collection_name": profile_collection_name},
+        )
+
+    def _get_profile_tool_kwargs(self, raise_exception: bool) -> dict:
+        """Shared profile tool configuration."""
+        return {
+            "profile_dir": self.profile_dir,
+            "profile_backend": self.profile_backend,
+            "profile_store_name": self.profile_store_name,
+            "profile_max_capacity": self.profile_max_capacity,
+            "raise_exception": raise_exception,
+        }
+
     async def summarize_memory(
         self,
         messages: list[Message | dict],
@@ -209,6 +284,7 @@ class ReMe(Application):
             format_messages.append(message)
 
         if version == "default":
+            profile_tool_kwargs = self._get_profile_tool_kwargs(raise_exception)
             personal_summarizer_tools: list = [
                 AddDraftAndRetrieveSimilarMemory(
                     enable_thinking_params=enable_thinking_params,
@@ -227,20 +303,28 @@ class ReMe(Application):
                 ),
             ]
             if self.enable_profile:
+                if self.profile_backend == "vector":
+                    profile_context_tool = RetrieveProfile(
+                        top_k=min(5, retrieve_top_k),
+                        enable_thinking_params=False,
+                        enable_memory_target=False,
+                        enable_multiple=False,
+                        **profile_tool_kwargs,
+                    )
+                else:
+                    profile_context_tool = ReadAllProfiles(
+                        enable_thinking_params=False,
+                        enable_memory_target=False,
+                        **profile_tool_kwargs,
+                    )
                 personal_summarizer_tools.extend(
                     [
-                        ReadAllProfiles(
-                            enable_thinking_params=False,
-                            enable_memory_target=False,
-                            profile_dir=self.profile_dir,
-                            raise_exception=raise_exception,
-                        ),
+                        profile_context_tool,
                         UpdateProfilesV1(
                             enable_thinking_params=enable_thinking_params,
                             enable_memory_target=False,
                             enable_multiple=True,
-                            profile_dir=self.profile_dir,
-                            raise_exception=raise_exception,
+                            **profile_tool_kwargs,
                         ),
                     ],
                 )
@@ -379,16 +463,24 @@ class ReMe(Application):
         self._ensure_started()
 
         if version == "default":
+            profile_tool_kwargs = self._get_profile_tool_kwargs(raise_exception)
             personal_retriever_tools = []
             if self.enable_profile:
-                personal_retriever_tools.append(
-                    ReadAllProfiles(
+                if self.profile_backend == "vector":
+                    profile_context_tool = RetrieveProfile(
+                        top_k=min(5, retrieve_top_k),
                         enable_thinking_params=False,
                         enable_memory_target=False,
-                        profile_dir=self.profile_dir,
-                        raise_exception=raise_exception,
-                    ),
-                )
+                        enable_multiple=False,
+                        **profile_tool_kwargs,
+                    )
+                else:
+                    profile_context_tool = ReadAllProfiles(
+                        enable_thinking_params=False,
+                        enable_memory_target=False,
+                        **profile_tool_kwargs,
+                    )
+                personal_retriever_tools.append(profile_context_tool)
             personal_retriever_tools.extend(
                 [
                     RetrieveMemory(
@@ -507,6 +599,34 @@ class ReMe(Application):
 
         return self._unwrap_memory_result(result, "retrieve_memory", return_dict)
 
+    async def retrieve_profile(
+        self,
+        query: str | list[str],
+        user_name: str,
+        top_k: int = 5,
+        return_dict: bool = False,
+    ) -> str | dict:
+        """Retrieve relevant profile rows for a user."""
+        self._ensure_started()
+        if not self.enable_profile:
+            raise RuntimeError("Profile functionality is disabled.")
+
+        profile_handler = self.get_profile_handler(user_name)
+        if profile_handler is None:
+            raise RuntimeError("Profile functionality is disabled.")
+
+        retrieved_nodes, output = await profile_handler.aretrieve(
+            query=query,
+            limit=top_k,
+            add_profile_id=True,
+            add_history_id=True,
+        )
+        result = {
+            "answer": output or "No matching profiles found.",
+            "retrieved_nodes": retrieved_nodes,
+        }
+        return self._unwrap_memory_result(result, "retrieve_profile", return_dict)
+
     async def add_memory(
         self,
         memory_content: str,
@@ -673,15 +793,23 @@ class ReMe(Application):
     @property
     def profile_path(self) -> Path | None:
         """Get the path to the profile directory. Returns None if profile is disabled."""
-        if not self.enable_profile:
+        if not self.enable_profile or self.profile_backend != "filesystem":
             return None
-        return Path(self.profile_dir) / self.default_vector_store.collection_name
+        collection_name = self.service_context.service_config.vector_stores["default"].collection_name
+        return Path(self.profile_dir) / collection_name
 
     def get_profile_handler(self, user_name: str) -> ProfileHandler | None:
         """Get the profile handler for the specified user. Returns None if profile is disabled."""
         if not self.enable_profile:
             return None
-        return ProfileHandler(memory_target=user_name, profile_path=self.profile_path)
+        return ProfileHandler(
+            memory_target=user_name,
+            profile_path=self.profile_path,
+            service_context=self.service_context,
+            profile_backend=self.profile_backend,
+            profile_store_name=self.profile_store_name,
+            max_capacity=self.profile_max_capacity,
+        )
 
 
 def main():

reme/reme_light.py

@@ -119,7 +119,7 @@ class ReMeLight(Application):
             The following directory structure will be created:
                 - {working_dir}/           - Root working directory
                 - {working_dir}/memory/    - Memory storage files
-                - {working_dir}/tool_result/ - Compacted tool result files
+                - {working_dir}/tool_results/ - Compacted tool result files
                 - {working_dir}/dialog/    - Raw conversation records
         """
         # Initialize working directory structure
@@ -127,7 +127,7 @@ class ReMeLight(Application):
         self.working_path.mkdir(parents=True, exist_ok=True)
         self.memory_path = self.working_path / "memory"
         self.memory_path.mkdir(parents=True, exist_ok=True)
-        self.tool_result_path = self.working_path / "tool_result"
+        self.tool_result_path = self.working_path / "tool_results"
         self.tool_result_path.mkdir(parents=True, exist_ok=True)
         self.dialog_path = self.working_path / "dialog"
         self.dialog_path.mkdir(parents=True, exist_ok=True)
@@ -135,12 +135,14 @@ class ReMeLight(Application):
         self.vector_weight: float = vector_weight
         self.candidate_multiplier: float = candidate_multiplier
 
-        # Build the file watcher config: use provided watch_paths if given, otherwise use defaults
-        _default_watch_paths = [
-            str(self.working_path / "MEMORY.md"),
-            str(self.working_path / "memory.md"),
-            str(self.memory_path),
-        ]
+        # Pick the existing memory markdown file. On case-insensitive filesystems
+        # (Windows NTFS, macOS APFS/HFS+) ``MEMORY.md`` and ``memory.md`` are the
+        # same file, so this also avoids watching it twice. Default to ``MEMORY.md``
+        # when neither exists yet.
+        _memory_md = self.working_path / "MEMORY.md"
+        if not _memory_md.exists() and (self.working_path / "memory.md").exists():
+            _memory_md = self.working_path / "memory.md"
+        _default_watch_paths = [str(_memory_md), str(self.memory_path)]
         if default_file_watcher_config and default_file_watcher_config.get("watch_paths"):
             _merged_file_watcher_config = default_file_watcher_config
         else:

reme4/__init__.py

@@ -0,0 +1,26 @@
+"""ReMe CLI package."""
+
+__version__ = "0.4.0.0"
+
+from . import config
+from . import constants
+from . import enumeration
+from . import schema
+from . import steps
+from . import utils
+from .application import Application
+from .components import BaseComponent
+from .reme import ReMe
+
+__all__ = [
+    "Application",
+    "BaseComponent",
+    "ReMe",
+    # submodules
+    "config",
+    "constants",
+    "enumeration",
+    "schema",
+    "steps",
+    "utils",
+]

reme4/application.py

@@ -0,0 +1,174 @@
+"""Main application entry point."""
+
+import asyncio
+import heapq
+from pathlib import Path
+from typing import AsyncGenerator
+
+from .components import BaseComponent, ApplicationContext
+from .enumeration import ComponentEnum
+from .schema import Response, StreamChunk
+from .utils import execute_stream_task, print_logo, get_logger
+
+
+class Application(BaseComponent):
+    """Main application: initializes components, resolves dependencies, runs jobs."""
+
+    def __init__(self, **kwargs) -> None:
+        self.context = ApplicationContext(**kwargs)
+
+        working_path = Path(self.config.working_dir).absolute()
+        working_path.mkdir(parents=True, exist_ok=True)
+        (working_path / self.config.metadata_dir).mkdir(parents=True, exist_ok=True)
+        (working_path / self.config.daily_dir).mkdir(parents=True, exist_ok=True)
+        (working_path / self.config.knowledge_dir).mkdir(parents=True, exist_ok=True)
+
+        if self.config.enable_logo:
+            print_logo(self.config)
+
+        logger = get_logger(
+            log_to_console=self.config.log_to_console,
+            log_to_file=self.config.log_to_file,
+            force_init=True,
+        )
+        logger.info(f"Initializing {self.config.app_name} Application")
+        super().__init__()
+
+        from .components import R
+
+        # Service
+        service_config = self.config.service
+        if not service_config.backend:
+            raise ValueError("Service configuration is missing the required 'backend' field")
+        service_cls = R.get(ComponentEnum.SERVICE, service_config.backend)
+        if not service_cls:
+            raise ValueError(f"Unregistered service backend '{service_config.backend}'")
+        params = service_config.model_dump()
+        params["app_context"] = self.context
+        self.context.service = service_cls(**params)
+
+        # Components
+        for component_type, component_configs in self.config.components.items():
+            self.context.components[component_type] = {}
+            for name, config in component_configs.items():
+                if not config.backend:
+                    raise ValueError(f"Component '{name}' is missing the required 'backend' field")
+                backend_cls = R.get(component_type, config.backend)
+                if not backend_cls:
+                    raise ValueError(f"Unregistered backend '{config.backend}' for component '{name}'")
+                params = config.model_dump()
+                params.setdefault("name", name)
+                params["app_context"] = self.context
+                self.context.components[component_type][name] = backend_cls(**params)
+
+        # Jobs
+        for job_config in self.config.jobs:
+            if not job_config.backend:
+                raise ValueError(f"Job '{job_config.name}' is missing the required 'backend' field")
+            job_cls = R.get(ComponentEnum.JOB, job_config.backend)
+            if not job_cls:
+                raise ValueError(f"Unregistered backend '{job_config.backend}' for job '{job_config.name}'")
+            params = job_config.model_dump()
+            params["app_context"] = self.context
+            self.context.jobs[job_config.name] = job_cls(**params)
+
+    @property
+    def config(self):
+        """Application configuration."""
+        return self.context.app_config
+
+    def _topological_order(self) -> list[BaseComponent]:
+        """Kahn's algorithm. Raises on missing required dep or cycle."""
+        nodes: dict[tuple[ComponentEnum, str], BaseComponent] = {
+            (ctype, name): comp for ctype, group in self.context.components.items() for name, comp in group.items()
+        }
+
+        in_degree: dict[tuple[ComponentEnum, str], int] = dict.fromkeys(nodes, 0)
+        dependents: dict[tuple[ComponentEnum, str], list[tuple[ComponentEnum, str]]] = {k: [] for k in nodes}
+        for key, comp in nodes.items():
+            for dep in comp.dependencies:
+                dep_key = (dep.ctype, dep.name)
+                if dep_key in nodes:
+                    dependents[dep_key].append(key)
+                    in_degree[key] += 1
+                elif not dep.optional:
+                    raise ValueError(
+                        f"Component {key[0].value}:{key[1]} depends on {dep.ctype.value}:{dep.name}, not registered",
+                    )
+
+        ready = [k for k, d in in_degree.items() if d == 0]
+        heapq.heapify(ready)
+        ordered: list[BaseComponent] = []
+        while ready:
+            key = heapq.heappop(ready)
+            ordered.append(nodes[key])
+            for downstream in dependents[key]:
+                in_degree[downstream] -= 1
+                if in_degree[downstream] == 0:
+                    heapq.heappush(ready, downstream)
+
+        if len(ordered) != len(nodes):
+            unresolved = [f"{k[0].value}:{k[1]}" for k, d in in_degree.items() if d > 0]
+            raise ValueError(f"Circular dependency detected among: {unresolved}")
+        return ordered
+
+    async def _start(self) -> None:
+        """Start components in topological order, then jobs."""
+        start_order = self._topological_order()
+        order_str = " -> ".join(f"{c.component_type.value}:{c.name}" for c in start_order)
+        self.logger.info(f"Component start order: {order_str}")
+
+        for component in start_order:
+            try:
+                await component.start()
+            except Exception as e:
+                self.logger.exception(f"Failed to start {component.component_type.value}:{component.name}: {e}")
+
+        for name, job in self.context.jobs.items():
+            try:
+                await job.start()
+            except Exception as e:
+                self.logger.exception(f"Failed to start job '{name}': {e}")
+
+    async def _close(self) -> None:
+        """Close all jobs, then components in reverse."""
+        for name, job in self.context.jobs.items():
+            try:
+                await job.close()
+            except Exception as e:
+                self.logger.exception(f"Failed to close job '{name}': {e}")
+
+        for components in self.context.components.values():
+            for component in components.values():
+                try:
+                    await component.close()
+                except Exception as e:
+                    self.logger.exception(f"Failed to close {component.component_type.value}:{component.name}: {e}")
+
+    async def run_job(self, name: str, /, **kwargs) -> Response:
+        """Execute a registered job by name."""
+        if name not in self.context.jobs:
+            raise KeyError(f"Job '{name}' not found")
+        return await self.context.jobs[name](**kwargs)
+
+    async def run_stream_job(self, name: str, /, **kwargs) -> AsyncGenerator[StreamChunk, None]:
+        """Execute a streaming job and yield chunks."""
+        if name not in self.context.jobs:
+            raise KeyError(f"Job '{name}' not found")
+        job = self.context.jobs[name]
+        stream_queue = asyncio.Queue()
+        task = asyncio.create_task(job(stream_queue=stream_queue, **kwargs))
+        async for chunk in execute_stream_task(
+            stream_queue=stream_queue,
+            task=task,
+            task_name=name,
+            output_format="chunk",
+        ):
+            assert isinstance(chunk, StreamChunk)
+            yield chunk
+
+    def run_app(self):
+        """Start the service and serve the application."""
+        if self.context.service is None:
+            raise RuntimeError("Service not configured")
+        self.context.service.run_app(app=self)

reme4/components/__init__.py

@@ -0,0 +1,43 @@
+"""Components"""
+
+from . import as_llm
+from . import as_llm_formatter
+from . import as_token_counter
+from . import client
+from . import embedding
+from . import file_graph
+from . import file_parser
+from . import file_store
+from . import file_watcher
+from . import job
+from . import keyword_index
+from . import service
+from . import tokenizer
+from .application_context import ApplicationContext
+from .base_component import BaseComponent
+from .component_registry import ComponentRegistry, R
+from .prompt_handler import PromptHandler
+from .runtime_context import RuntimeContext
+
+__all__ = [
+    "ApplicationContext",
+    "BaseComponent",
+    "ComponentRegistry",
+    "R",
+    "PromptHandler",
+    "RuntimeContext",
+    # base components
+    "as_llm",
+    "as_llm_formatter",
+    "as_token_counter",
+    "client",
+    "embedding",
+    "file_graph",
+    "file_parser",
+    "file_store",
+    "file_watcher",
+    "job",
+    "keyword_index",
+    "service",
+    "tokenizer",
+]

reme4/components/application_context.py

@@ -0,0 +1,28 @@
+"""Application context: shared state container for components, jobs, and service."""
+
+from ..enumeration import ComponentEnum
+from ..schema import ApplicationConfig
+
+
+class ApplicationContext:
+    """Holds the parsed config and instantiated components, jobs, and service.
+
+    Acts as a passive state container. The actual wiring (resolving backends from
+    the registry and instantiating each component) is performed by Application.
+    """
+
+    def __init__(self, **kwargs):
+        # Parse and validate raw config kwargs into a typed ApplicationConfig.
+        self.app_config: ApplicationConfig = ApplicationConfig(**kwargs)
+
+        # Local imports to avoid circular dependencies during module init.
+        from .base_component import BaseComponent
+        from .job import BaseJob
+        from .service import BaseService
+
+        # Service endpoint (e.g. HTTP/MCP). Populated by Application.__init__.
+        self.service: BaseService | None = None
+        # Components keyed by type then by user-defined name.
+        self.components: dict[ComponentEnum, dict[str, BaseComponent]] = {}
+        # Jobs keyed by user-defined name.
+        self.jobs: dict[str, BaseJob] = {}

reme4/components/as_llm/__init__.py

@@ -0,0 +1,53 @@
+"""AgentScope LLM model wrappers."""
+
+from agentscope.model import AnthropicChatModel, ChatModelBase, OpenAIChatModel
+
+from ..base_component import BaseComponent
+from ..component_registry import R
+from ...enumeration import ComponentEnum
+
+
+class BaseAsLLM(BaseComponent):
+    """Base wrapper for AgentScope chat models. Builds ``self.model`` in ``_start``."""
+
+    component_type = ComponentEnum.AS_LLM
+
+    def __init__(self, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self.model: ChatModelBase | None = None
+
+    async def _close(self) -> None:
+        self.model = None
+
+
+@R.register("openai")
+class OpenAIAsLLM(BaseAsLLM):
+    """OpenAI chat model wrapper."""
+
+    async def _start(self) -> None:
+        self.model = OpenAIChatModel(**self.kwargs)
+
+    async def _close(self) -> None:
+        if self.model is not None:
+            assert isinstance(self.model, OpenAIChatModel)
+            await self.model.client.close()
+
+
+@R.register("anthropic")
+class AnthropicAsLLM(BaseAsLLM):
+    """Anthropic chat model wrapper."""
+
+    async def _start(self) -> None:
+        self.model = AnthropicChatModel(**self.kwargs)
+
+    async def _close(self) -> None:
+        if self.model is not None:
+            assert isinstance(self.model, AnthropicChatModel)
+            await self.model.client.close()
+
+
+__all__ = [
+    "BaseAsLLM",
+    "OpenAIAsLLM",
+    "AnthropicAsLLM",
+]

reme4/components/as_llm_formatter/__init__.py

@@ -0,0 +1,44 @@
+"""AgentScope LLM formatter wrappers."""
+
+from agentscope.formatter import AnthropicChatFormatter, FormatterBase
+
+from .reme_openai_chat_formatter import ReMeOpenAIChatFormatter
+from ..base_component import BaseComponent
+from ..component_registry import R
+from ...enumeration import ComponentEnum
+
+
+class BaseAsLLMFormatter(BaseComponent):
+    """Base wrapper for AgentScope formatters. Builds ``self.formatter`` in ``_start``."""
+
+    component_type = ComponentEnum.AS_LLM_FORMATTER
+
+    def __init__(self, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self.formatter: FormatterBase | None = None
+
+    async def _close(self) -> None:
+        self.formatter = None
+
+
+@R.register("openai")
+class AsOpenAIChatFormatter(BaseAsLLMFormatter):
+    """OpenAI chat formatter wrapper (uses ReMe extensions)."""
+
+    async def _start(self) -> None:
+        self.formatter = ReMeOpenAIChatFormatter(**self.kwargs)
+
+
+@R.register("anthropic")
+class AsAnthropicChatFormatter(BaseAsLLMFormatter):
+    """Anthropic chat formatter wrapper."""
+
+    async def _start(self) -> None:
+        self.formatter = AnthropicChatFormatter(**self.kwargs)
+
+
+__all__ = [
+    "BaseAsLLMFormatter",
+    "AsOpenAIChatFormatter",
+    "AsAnthropicChatFormatter",
+]

reme4/components/as_llm_formatter/reme_openai_chat_formatter.py

@@ -0,0 +1,141 @@
+"""OpenAI chat formatter with ReMe extensions: image promotion and reasoning_content."""
+
+import json
+from typing import Any
+
+from agentscope.formatter import OpenAIChatFormatter
+
+# noinspection PyProtectedMember
+from agentscope.formatter._openai_formatter import (
+    _format_openai_image_block,
+    _to_openai_audio_data,
+)
+from agentscope.message import Msg, TextBlock, ImageBlock, URLSource
+
+
+def _format_openai_video_block(video_block: dict) -> dict[str, Any]:
+    """Convert a video block to OpenAI ``video_url`` content."""
+    source = video_block["source"]
+    if source["type"] == "url":
+        url = source["url"]
+    elif source["type"] == "base64":
+        url = f"data:{source['media_type']};base64,{source['data']}"
+    else:
+        raise ValueError(f"Unsupported video source type: {source['type']}")
+    return {"type": "video_url", "video_url": {"url": url}}
+
+
+class ReMeOpenAIChatFormatter(OpenAIChatFormatter):
+    """OpenAIChatFormatter + tool-result image promotion + reasoning_content passthrough."""
+
+    async def _format(self, msgs: list[Msg]) -> list[dict[str, Any]]:
+        """Format ``Msg`` list into OpenAI chat-completion message dicts."""
+        self.assert_list_of_msgs(msgs)
+
+        messages: list[dict] = []
+        i = 0
+        while i < len(msgs):
+            msg = msgs[i]
+            content_blocks = []
+            tool_calls = []
+            reasoning_content_blocks = []
+
+            for block in msg.get_content_blocks():
+                typ = block.get("type")
+
+                if typ == "text":
+                    content_blocks.append({**block})
+
+                elif typ == "thinking":
+                    reasoning_content_blocks.append({**block})
+
+                elif typ == "tool_use":
+                    tool_calls.append(
+                        {
+                            "id": block.get("id"),
+                            "type": "function",
+                            "function": {
+                                "name": block.get("name"),
+                                "arguments": json.dumps(block.get("input", {}), ensure_ascii=False),
+                            },
+                        },
+                    )
+
+                elif typ == "tool_result":
+                    textual_output, multimodal_data = self.convert_tool_result_to_string(block["output"])
+                    messages.append(
+                        {
+                            "role": "tool",
+                            "tool_call_id": block.get("id"),
+                            "content": textual_output,
+                            "name": block.get("name"),
+                        },
+                    )
+
+                    # OpenAI tool messages can't carry images; promote to a follow-up user message.
+                    promoted_blocks = []
+                    for url, multimodal_block in multimodal_data:
+                        if multimodal_block["type"] == "image" and self.promote_tool_result_images:
+                            promoted_blocks.extend(
+                                [
+                                    TextBlock(type="text", text=f"\n- The image from '{url}': "),
+                                    ImageBlock(type="image", source=URLSource(type="url", url=url)),
+                                ],
+                            )
+
+                    if promoted_blocks:
+                        promoted_blocks = [
+                            TextBlock(
+                                type="text",
+                                text="<system-info>The following are the image contents from the tool "
+                                f"result of '{block['name']}':",
+                            ),
+                            *promoted_blocks,
+                            TextBlock(type="text", text="</system-info>"),
+                        ]
+                        msgs.insert(
+                            i + 1,
+                            Msg(name="user", content=promoted_blocks, role="user"),
+                        )
+
+                elif typ == "image":
+                    content_blocks.append(_format_openai_image_block(block))
+
+                elif typ == "audio":
+                    # Skip assistant audio — not a valid input modality.
+                    if msg.role == "assistant":
+                        continue
+                    content_blocks.append(
+                        {
+                            "type": "input_audio",
+                            "input_audio": _to_openai_audio_data(block["source"]),
+                        },
+                    )
+
+                elif typ == "video":
+                    # Skip assistant video — not a valid input modality.
+                    if msg.role == "assistant":
+                        continue
+                    content_blocks.append(_format_openai_video_block(block))
+
+            msg_openai = {
+                "role": msg.role,
+                "name": msg.name,
+                "content": content_blocks or None,
+            }
+
+            if tool_calls:
+                msg_openai["tool_calls"] = tool_calls
+
+            # Merge thinking blocks into reasoning_content for compatible models.
+            if reasoning_content_blocks:
+                reasoning_msg = "\n".join(r.get("thinking", "") for r in reasoning_content_blocks)
+                if reasoning_msg:
+                    msg_openai["reasoning_content"] = reasoning_msg
+
+            if msg_openai["content"] or msg_openai.get("tool_calls"):
+                messages.append(msg_openai)
+
+            i += 1
+
+        return messages

reme4/components/as_token_counter/__init__.py

@@ -0,0 +1,35 @@
+"""AgentScope token counter wrappers."""
+
+from agentscope.token import TokenCounterBase
+
+from .estimate_token_counter import EstimatedTokenCounter
+from ..base_component import BaseComponent
+from ..component_registry import R
+from ...enumeration import ComponentEnum
+
+
+class BaseAsTokenCounter(BaseComponent):
+    """Base wrapper for AgentScope token counters. Builds ``self.token_counter`` in ``_start``."""
+
+    component_type = ComponentEnum.AS_TOKEN_COUNTER
+
+    def __init__(self, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self.token_counter: TokenCounterBase | None = None
+
+    async def _close(self) -> None:
+        self.token_counter = None
+
+
+@R.register("estimated")
+class EstimatedAsTokenCounter(BaseAsTokenCounter):
+    """Character-based estimated token counter — fast but approximate."""
+
+    async def _start(self) -> None:
+        self.token_counter = EstimatedTokenCounter(**self.kwargs)
+
+
+__all__ = [
+    "BaseAsTokenCounter",
+    "EstimatedAsTokenCounter",
+]

reme4/components/as_token_counter/estimate_token_counter.py

@@ -0,0 +1,21 @@
+"""Character-based token-count estimator."""
+
+from agentscope.token import TokenCounterBase
+
+
+class EstimatedTokenCounter(TokenCounterBase):
+    """Approximate token count as ``encoded_byte_len / divisor``.
+
+    Cheap proxy when exact counts aren't needed; use the model's real
+    tokenizer for accuracy.
+    """
+
+    def __init__(self, estimate_divisor: float = 4, encoding: str = "utf-8"):
+        if estimate_divisor <= 0:
+            raise ValueError("estimate_divisor must be positive")
+        self.estimate_divisor: float = estimate_divisor
+        self.encoding: str = encoding
+
+    async def count(self, text: str, **_kwargs) -> int:
+        """Estimated token count for ``text``."""
+        return int(len(text.encode(self.encoding)) / self.estimate_divisor + 0.5)

reme4/components/base_component.py

@@ -0,0 +1,185 @@
+"""Base class for components."""
+
+import asyncio
+from abc import ABC
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast
+
+from ..enumeration import ComponentEnum
+from ..utils import get_logger
+
+if TYPE_CHECKING:
+    from .application_context import ApplicationContext
+
+T = TypeVar("T", bound="BaseComponent")
+
+
+class Dependency:
+    """Declared dependency: bind() return value, instance attribute placeholder, and topological-sort edge."""
+
+    __slots__ = ("ctype", "name", "default_factory", "optional")
+
+    def __init__(
+        self,
+        ctype: ComponentEnum,
+        name: str,
+        default_factory: Callable[[], Any] | None = None,
+        optional: bool = True,
+    ) -> None:
+        self.ctype = ctype
+        self.name = name
+        self.default_factory = default_factory
+        self.optional = optional
+
+    def __repr__(self) -> str:
+        suffix = "?" if self.optional else ""
+        return f"<unresolved {self.ctype.value}:{self.name}{suffix}>"
+
+    def __getattr__(self, item: str) -> Any:
+        # Guard against using the dependency before start() resolves it.
+        raise RuntimeError(
+            f"Dependency {self.ctype.value}:{self.name} accessed before start() (attribute '{item}')",
+        )
+
+
+class BaseComponent(ABC):
+    """Async lifecycle base class with bind-based dependency injection."""
+
+    component_type = ComponentEnum.BASE
+
+    def __init__(
+        self,
+        name: str | None = None,
+        backend: str = "",
+        app_context: "ApplicationContext | None" = None,
+        **kwargs,
+    ) -> None:
+        self.name: str = name or self.__class__.__name__
+        self.backend: str = backend
+        self.app_context: "ApplicationContext | None" = app_context
+        self.kwargs: dict = dict(kwargs)
+        self.logger = get_logger()
+        if hasattr(self.logger, "bind"):
+            self.logger = self.logger.bind(component=self.name)
+
+        self._is_started: bool = False
+        self._lock: asyncio.Lock = asyncio.Lock()
+        # Components created from bind() default_factory in standalone mode (auto-managed lifecycle).
+        self._owned: list["BaseComponent"] = []
+
+    @property
+    def is_started(self) -> bool:
+        """Whether the component has been started."""
+        return self._is_started
+
+    # ----- Dependency declaration ----------------------------------------
+
+    @staticmethod
+    def bind(
+        name: str | None,
+        base_cls: type[T],
+        *,
+        default_factory: Callable[[], T] | None = None,
+        optional: bool = True,
+    ) -> T | None:
+        """Declare a dependency on another component; resolved at start(). Empty name → None."""
+        if not name:
+            return None
+        ctype = getattr(base_cls, "component_type", None)
+        if not isinstance(ctype, ComponentEnum) or ctype is ComponentEnum.BASE:
+            raise TypeError(f"{base_cls.__name__} must declare a non-BASE ComponentEnum 'component_type'")
+        return cast(T, Dependency(ctype, name, default_factory, optional))
+
+    @property
+    def dependencies(self) -> list[Dependency]:
+        """All unresolved bindings declared on this instance."""
+        return [v for v in self.__dict__.values() if isinstance(v, Dependency)]
+
+    async def _resolve_bindings(self) -> None:
+        """Replace Dependency placeholders with real components (or default_factory / None for optional)."""
+        for attr, value in list(self.__dict__.items()):
+            if not isinstance(value, Dependency):
+                continue
+            if self.app_context is None:
+                # Standalone mode: factory or (optional → None) or keep placeholder.
+                if value.default_factory is not None:
+                    instance = value.default_factory()
+                    setattr(self, attr, instance)
+                    if isinstance(instance, BaseComponent):
+                        self._owned.append(instance)
+                elif value.optional:
+                    setattr(self, attr, None)
+            else:
+                target = self.app_context.components.get(value.ctype, {}).get(value.name)
+                if target is not None:
+                    setattr(self, attr, target)
+                elif value.optional:
+                    setattr(self, attr, None)
+                else:
+                    raise ValueError(f"{value.ctype.value} '{value.name}' not found.")
+
+    # ----- Lookup --------------------------------------------------------
+
+    @property
+    def working_path(self) -> Path:
+        """Resolved working directory from app context or cwd."""
+        if self.app_context is None:
+            return Path.cwd()
+        return Path(self.app_context.app_config.working_dir)
+
+    @property
+    def working_metadata_path(self) -> Path:
+        """Resolved metadata directory: working_path / metadata_dir, or absolute metadata_dir."""
+        if self.app_context is None:
+            return Path.cwd() / "metadata"
+        return self.working_path / self.app_context.app_config.metadata_dir
+
+    # ----- Lifecycle -----------------------------------------------------
+
+    async def _start(self) -> None:
+        """Subclass hook: start logic."""
+
+    async def _close(self) -> None:
+        """Subclass hook: close logic."""
+
+    async def dump(self) -> None:
+        """Persist in-memory state to disk. Override in subclasses that need persistence."""
+
+    async def load(self) -> None:
+        """Restore in-memory state from disk. Override in subclasses that need persistence."""
+
+    async def start(self) -> None:
+        """Resolve bindings → start owned fallbacks → _start(). No-op if already started."""
+        async with self._lock:
+            if self._is_started:
+                return
+            await self._resolve_bindings()
+            for owned in self._owned:
+                await owned.start()
+            await self._start()
+            self._is_started = True
+
+    async def close(self) -> None:
+        """_close() → close owned fallbacks in reverse. No-op if not started."""
+        async with self._lock:
+            if not self._is_started:
+                return
+            await self._close()
+            for owned in reversed(self._owned):
+                await owned.close()
+            self._is_started = False
+
+    async def restart(self) -> None:
+        """Close then start."""
+        await self.close()
+        await self.start()
+
+    async def __call__(self, **kwargs):
+        raise NotImplementedError
+
+    async def __aenter__(self) -> "BaseComponent":
+        await self.start()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb) -> None:
+        await self.close()

reme4/components/client/__init__.py

@@ -0,0 +1,7 @@
+"""Client components."""
+
+from .base_client import BaseClient
+from .http_client import HttpClient
+from .mcp_client import MCPClient
+
+__all__ = ["BaseClient", "HttpClient", "MCPClient"]

reme4/components/client/base_client.py

@@ -0,0 +1,41 @@
+"""Base client abstraction."""
+
+import json
+from abc import abstractmethod
+from collections.abc import AsyncGenerator
+
+from ..base_component import BaseComponent
+from ...enumeration import ComponentEnum
+
+
+class BaseClient(BaseComponent):
+    """Abstract base for clients that communicate with ReMe services."""
+
+    component_type = ComponentEnum.CLIENT
+
+    def __init__(self, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self.client = None
+
+    async def _start(self) -> None:
+        """Initialize the client."""
+
+    async def _close(self) -> None:
+        """Close the client and release resources."""
+
+    @abstractmethod
+    def _execute(self) -> AsyncGenerator[str, None]:
+        """Backend-specific execution; yield text chunks (single yield for non-streaming backends)."""
+
+    @abstractmethod
+    async def list_actions(self) -> list[dict]:
+        """Discover available actions on the server; each dict is the raw backend descriptor."""
+
+    async def __call__(self) -> AsyncGenerator[str, None]:
+        """Dispatch: action='list' returns the action catalog; otherwise delegate to _execute()."""
+        if getattr(self, "action", None) == "list":
+            actions = await self.list_actions()
+            yield json.dumps(actions, indent=2, ensure_ascii=False)
+            return
+        async for chunk in self._execute():
+            yield chunk

reme4/components/client/http_client.py

@@ -0,0 +1,143 @@
+"""HTTP client for ReMe services."""
+
+import json
+import os
+from collections.abc import AsyncGenerator
+
+import httpx
+
+from .base_client import BaseClient
+from ..component_registry import R
+from ...constants import REME_SERVICE_INFO, REME_DEFAULT_HOST, REME_DEFAULT_PORT
+from ...enumeration import ChunkEnum
+from ...schema import StreamChunk
+
+
+@R.register("http")
+class HttpClient(BaseClient):
+    """HTTP client that auto-adapts to JSON or SSE endpoints via Content-Type."""
+
+    def __init__(
+        self,
+        action: str,
+        host: str | None = None,
+        port: int | None = None,
+        timeout: float = 30.0,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        # Resolve host/port: explicit args > env var > defaults
+        if not (host and port):
+            if service_info := os.environ.get(REME_SERVICE_INFO):
+                try:
+                    data = json.loads(service_info)
+                    host = data["host"]
+                    port = data["port"]
+                except Exception:
+                    self.logger.warning(f"Invalid service info: {service_info}")
+                    host, port = REME_DEFAULT_HOST, REME_DEFAULT_PORT
+            else:
+                host, port = REME_DEFAULT_HOST, REME_DEFAULT_PORT
+
+        self.action = action
+        self.base_url = f"http://{host}:{port}"
+        self.timeout = timeout
+
+    async def _start(self) -> None:
+        """Initialize the HTTP client."""
+        if self.client is None:
+            self.client = httpx.AsyncClient(base_url=self.base_url, timeout=self.timeout)
+
+    async def _iter_stream_chunks(self) -> AsyncGenerator[StreamChunk, None]:
+        """Send request and yield raw StreamChunks; auto-detects JSON vs SSE via Content-Type.
+
+        For JSON responses: yields a single CONTENT chunk with the raw response body.
+        For SSE responses: yields each streaming chunk as it arrives.
+        """
+        if self.client is None:
+            raise RuntimeError("Client not initialized. Call _start() first.")
+
+        async with self.client.stream("POST", f"/{self.action}", json=self.kwargs) as resp:
+            resp.raise_for_status()
+            ctype = resp.headers.get("content-type", "")
+
+            if ctype.startswith("text/event-stream"):
+                async for line in resp.aiter_lines():
+                    if not line.startswith("data:"):
+                        continue
+                    payload = line[len("data:") :]
+                    if payload.strip() == "[DONE]":
+                        return
+                    try:
+                        data = json.loads(payload)
+                    except json.JSONDecodeError:
+                        continue
+                    chunk = StreamChunk(**data)
+                    if chunk.chunk_type == ChunkEnum.ERROR:
+                        # Surface server-side errors as exceptions so callers don't
+                        # mistake error chunks for valid content.
+                        raise RuntimeError(str(chunk.chunk))
+                    if chunk.done:
+                        return
+                    yield chunk
+            else:
+                body = await resp.aread()
+                yield StreamChunk(chunk_type=ChunkEnum.CONTENT, chunk=body.decode())
+
+    async def stream_chunks(self) -> AsyncGenerator[StreamChunk, None]:
+        """HTTP-specific richer access: yield raw StreamChunk objects (no display formatting)."""
+        async for chunk in self._iter_stream_chunks():
+            yield chunk
+
+    async def list_actions(self) -> list[dict]:
+        """Return raw OpenAPI operations; each dict gets an `action` key (path without leading '/')."""
+        if self.client is None:
+            raise RuntimeError("Client not initialized. Call _start() first.")
+        resp = await self.client.get("/openapi.json")
+        resp.raise_for_status()
+        spec = resp.json()
+        actions: list[dict] = []
+        for path, methods in spec.get("paths", {}).items():
+            for method, op in methods.items():
+                actions.append({"action": path.lstrip("/"), "method": method.upper(), **op})
+        return actions
+
+    @staticmethod
+    def _format_for_display(text: str) -> str:
+        """Render a JSON response as human-friendly CLI text; pass through unrecognized payloads."""
+        try:
+            data = json.loads(text)
+        except (ValueError, json.JSONDecodeError):
+            return text
+        if not (isinstance(data, dict) and isinstance(data.get("answer"), str)):
+            return json.dumps(data, indent=2, ensure_ascii=False) if isinstance(data, (dict, list)) else text
+        d = dict(data)
+        answer = d.pop("answer")
+        success = d.pop("success", None)
+        metadata = d.pop("metadata", None)
+        parts = [answer]
+        status_pieces = []
+        if success is not None:
+            status_pieces.append("✅" if success else "❌")
+        if metadata:
+            status_pieces.append(json.dumps(metadata, ensure_ascii=False))
+        if status_pieces:
+            parts.append(" ".join(status_pieces))
+        if d:
+            parts.append(json.dumps(d, indent=2, ensure_ascii=False))
+        return "\n".join(parts)
+
+    # pylint: disable=invalid-overridden-method
+    async def _execute(self) -> AsyncGenerator[str, None]:
+        """Yield text chunks for CLI display; JSON responses are pretty-formatted."""
+        async for chunk in self._iter_stream_chunks():
+            payload = chunk.chunk
+            text = payload if isinstance(payload, str) else json.dumps(payload, ensure_ascii=False)
+            yield self._format_for_display(text)
+
+    async def _close(self) -> None:
+        """Close the HTTP client."""
+        if self.client is not None:
+            await self.client.aclose()
+            self.client = None

reme4/components/client/mcp_client.py

@@ -0,0 +1,125 @@
+"""MCP client for ReMe services."""
+
+import json
+import os
+from collections.abc import AsyncGenerator
+from typing import Any
+
+from fastmcp import Client
+from fastmcp.client import SSETransport, StdioTransport, StreamableHttpTransport
+from fastmcp.client.client import CallToolResult
+
+from .base_client import BaseClient
+from ..component_registry import R
+from ...constants import REME_SERVICE_INFO, REME_DEFAULT_HOST, REME_DEFAULT_PORT
+
+_TRANSPORT_MAP = {
+    "sse": SSETransport,
+    "stdio": StdioTransport,
+    "streamable-http": StreamableHttpTransport,
+}
+
+
+@R.register("mcp")
+class MCPClient(BaseClient):
+    """MCP client that communicates with ReMe MCP service via fastmcp.Client.
+
+    Usage:
+        # SSE (default)
+        client = MCPClient(action="my_tool", host="localhost", port=8000, query="hello")
+        async with client:
+            async for text in client():
+                print(text)
+
+        # Streamable HTTP
+        client = MCPClient(action="my_tool", transport="streamable-http", host="localhost", port=8000)
+
+        # Stdio
+        client = MCPClient(action="my_tool", transport="stdio", command="python", args=["server.py"])
+
+        # Custom transport object
+        from fastmcp.client import SSETransport
+        client = MCPClient(action="my_tool", transport=SSETransport(url="http://host:port/sse"))
+    """
+
+    def __init__(
+        self,
+        action: str,
+        transport: str | Any = "sse",
+        host: str | None = None,
+        port: int | None = None,
+        timeout: float = 30.0,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        if isinstance(transport, str) and transport not in _TRANSPORT_MAP:
+            raise ValueError(f"Unknown transport: {transport!r}, expected one of {list(_TRANSPORT_MAP)}")
+
+        if isinstance(transport, str) and transport != "stdio":
+            if not (host and port):
+                if service_info := os.environ.get(REME_SERVICE_INFO):
+                    try:
+                        data = json.loads(service_info)
+                        host = data["host"]
+                        port = data["port"]
+                    except Exception:
+                        self.logger.warning(f"Invalid service info: {service_info}")
+                        host, port = REME_DEFAULT_HOST, REME_DEFAULT_PORT
+                else:
+                    host, port = REME_DEFAULT_HOST, REME_DEFAULT_PORT
+            self.host = host
+            self.port = port
+
+        self.action = action
+        self.transport = transport
+        self.timeout = timeout
+
+    def _build_transport(self):
+        if not isinstance(self.transport, str):
+            return self.transport
+
+        cls = _TRANSPORT_MAP[self.transport]
+
+        if self.transport == "stdio":
+            command = self.kwargs.pop("command", "")
+            args = self.kwargs.pop("args", [])
+            return cls(command=command, args=args)
+
+        path = "/sse" if self.transport == "sse" else "/mcp"
+        url = f"http://{self.host}:{self.port}{path}"
+        return cls(url=url)
+
+    # pylint: disable=unnecessary-dunder-call
+    async def _start(self) -> None:
+        if self.client is None:
+            self.client = Client(self._build_transport(), timeout=self.timeout)
+            await self.client.__aenter__()
+
+    # pylint: disable=invalid-overridden-method
+    async def _execute(self) -> AsyncGenerator[str, None]:
+        if self.client is None:
+            raise RuntimeError("Client not initialized. Call _start() first.")
+
+        result: CallToolResult = await self.client.call_tool(self.action, self.kwargs)
+        yield self._extract_text(result)
+
+    async def list_actions(self) -> list[dict]:
+        """Return raw MCP Tool dumps; each dict gets an `action` key (the tool name)."""
+        if self.client is None:
+            raise RuntimeError("Client not initialized. Call _start() first.")
+        tools = await self.client.list_tools()
+        return [tool.model_dump() for tool in tools]
+
+    # pylint: disable=unnecessary-dunder-call
+    async def _close(self) -> None:
+        if self.client is not None:
+            await self.client.__aexit__(None, None, None)
+            self.client = None
+
+    @staticmethod
+    def _extract_text(result: CallToolResult) -> str:
+        for block in result.content:
+            if hasattr(block, "text"):
+                return block.text
+        return str(result.content)

reme4/components/component_registry.py

@@ -0,0 +1,77 @@
+"""Global registry mapping (ComponentEnum, name) -> component class."""
+
+from typing import Callable, TypeVar, cast
+
+from .base_component import BaseComponent
+from ..enumeration import ComponentEnum
+from ..utils import get_logger
+
+T = TypeVar("T", bound=BaseComponent)
+
+
+class ComponentRegistry:
+    """Two-level registry: component_type -> name -> class.
+
+    Supports both direct calls — ``R.register(MyClass, "name")`` — and
+    decorator usage — ``@R.register("name")``.
+    """
+
+    def __init__(self) -> None:
+        self._registry: dict[ComponentEnum, dict[str, type[BaseComponent]]] = {}
+        self.logger = get_logger()
+
+    def _do_register(self, cls: type[T], name: str) -> type[T]:
+        """Insert `cls` under its `component_type` group; warn on overwrite."""
+        component_type = getattr(cls, "component_type", None)
+        if not isinstance(component_type, ComponentEnum):
+            raise TypeError(f"{cls.__name__} must have a ComponentEnum 'component_type' attribute")
+        if not name:
+            raise ValueError("Component name cannot be empty")
+
+        group = self._registry.setdefault(component_type, {})
+        if name in group:
+            self.logger.warning(f"Component '{name}' already registered for {component_type}, overwriting")
+        group[name] = cls
+        return cls
+
+    def register(
+        self,
+        cls_or_name: type[T] | str,
+        name: str | None = None,
+    ) -> Callable[[type[T]], type[T]] | type[T]:
+        """Register a component class directly, or return a decorator that does so."""
+        # Direct mode: first arg is the class itself.
+        if isinstance(cls_or_name, type):
+            return self._do_register(cast(type[T], cls_or_name), name if name is not None else cls_or_name.__name__)
+
+        # Decorator mode: first arg is the registration name.
+        if not isinstance(cls_or_name, str):
+            raise TypeError(f"Expected a class or string, got {type(cls_or_name).__name__}")
+
+        def decorator(decorated_cls: type[T]) -> type[T]:
+            return self._do_register(decorated_cls, cls_or_name)
+
+        return decorator
+
+    def get(self, component_type: ComponentEnum, name: str) -> type[BaseComponent] | None:
+        """Look up a registered class; return None if not found."""
+        return self._registry.get(component_type, {}).get(name)
+
+    def get_all(self, component_type: ComponentEnum) -> dict[str, type[BaseComponent]]:
+        """Return a shallow copy of all classes registered under `component_type`."""
+        return dict(self._registry.get(component_type, {}))
+
+    def unregister(self, component_type: ComponentEnum, name: str) -> bool:
+        """Remove an entry; return True if it existed, False otherwise."""
+        if (group := self._registry.get(component_type)) and name in group:
+            del group[name]
+            return True
+        return False
+
+    def clear(self) -> None:
+        """Drop every registered entry."""
+        self._registry.clear()
+
+
+# Process-wide singleton used throughout the codebase.
+R = ComponentRegistry()

reme4/components/embedding/__init__.py

@@ -0,0 +1,6 @@
+"""Embedding model implementations."""
+
+from .base_embedding_model import BaseEmbeddingModel
+from .openai_embedding_model import OpenAIEmbeddingModel
+
+__all__ = ["BaseEmbeddingModel", "OpenAIEmbeddingModel"]

reme4/components/embedding/base_embedding_model.py

@@ -0,0 +1,214 @@
+"""Base embedding model with LRU cache and disk persistence."""
+
+import asyncio
+import hashlib
+import os
+from abc import abstractmethod
+from collections import OrderedDict
+from pathlib import Path
+
+import numpy as np
+
+from ..base_component import BaseComponent
+from ...enumeration import ComponentEnum
+from ...schema import EmbNode
+
+
+class BaseEmbeddingModel(BaseComponent):
+    """Embedding model with LRU cache and disk persistence."""
+
+    component_type = ComponentEnum.EMBEDDING_MODEL
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+        base_url: str | None = None,
+        model_name: str = "",
+        dimensions: int = 1024,
+        pass_dimensions: bool = False,
+        max_batch_size: int = 10,
+        max_input_length: int = 8192,
+        max_cache_size: int = 10000,
+        enable_cache: bool = True,
+        cache_version: str = "v1",
+        max_retries: int = 3,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.api_key = api_key or os.environ.get("EMBEDDING_API_KEY", "")
+        self.base_url = base_url or os.environ.get("EMBEDDING_BASE_URL", "")
+        self.model_name = model_name
+        self.dimensions = dimensions
+        self.pass_dimensions = pass_dimensions
+        self.max_batch_size = max_batch_size
+        self.max_input_length = max_input_length
+        self.max_cache_size = max_cache_size
+        self.enable_cache = enable_cache
+        self.cache_version = cache_version
+        self.max_retries = max_retries
+        self._embedding_cache: OrderedDict[str, np.ndarray] = OrderedDict()
+        self.is_healthy: bool = True
+
+    @property
+    def cache_path(self) -> Path:
+        """Disk path for the embedding cache file."""
+        return self.working_metadata_path / "embedding_cache" / f"{self.name}_{self.cache_version}.npz"
+
+    async def _start(self) -> None:
+        """Load cache from disk on startup."""
+        await self.load()
+
+    async def health_check(self, timeout: float = 2.0) -> bool:
+        """Probe the provider; sets and returns is_healthy."""
+        tag = f"[EMBEDDING HEALTH CHECK] name={self.name} model={self.model_name}"
+        try:
+            result = await asyncio.wait_for(self._get_embeddings(["ping"]), timeout=timeout)
+            if not result or result[0] is None:
+                raise RuntimeError("empty embedding")
+            self.is_healthy = True
+            self.logger.info(f"{tag} -> OK")
+        except asyncio.TimeoutError:
+            self.is_healthy = False
+            self.logger.error(f"{tag} -> FAIL timeout({timeout}s)")
+        except Exception as e:
+            self.is_healthy = False
+            self.logger.error(f"{tag} -> FAIL {type(e).__name__}: {e}")
+        return self.is_healthy
+
+    async def _close(self) -> None:
+        """Persist cache to disk on shutdown."""
+        await self.dump()
+
+    # -- Public API --
+
+    async def get_embedding(self, input_text: str, **kwargs) -> np.ndarray | None:
+        """Get embedding for a single text."""
+        results = await self.get_embeddings([input_text], **kwargs)
+        return results[0] if results else None
+
+    async def get_embeddings(self, input_text: list[str], **kwargs) -> list[np.ndarray | None]:
+        """Get embeddings for a list of texts, with caching and batching."""
+        truncated = [t[: self.max_input_length] for t in input_text]
+        results: list[np.ndarray | None] = [None] * len(truncated)
+        to_compute: list[tuple[int, str]] = []
+
+        # Split into cache hits and misses
+        for idx, text in enumerate(truncated):
+            cached = self._get_from_cache(text)
+            if cached is not None:
+                results[idx] = cached
+            else:
+                to_compute.append((idx, text))
+
+        # Batch-compute misses with retry
+        if to_compute:
+            for i in range(0, len(to_compute), self.max_batch_size):
+                batch = to_compute[i : i + self.max_batch_size]
+                indices = [idx for idx, _ in batch]
+                texts = [text for _, text in batch]
+
+                embeddings = None
+                for attempt in range(self.max_retries):
+                    try:
+                        embeddings = await self._get_embeddings(texts, **kwargs)
+                        if embeddings and len(embeddings) == len(texts):
+                            break
+                    except (TimeoutError, ConnectionError, OSError):
+                        if attempt < self.max_retries - 1:
+                            await asyncio.sleep(2**attempt)
+                    except Exception:
+                        self.logger.exception("Embedding request failed")
+                        break
+
+                if not embeddings or len(embeddings) != len(texts):
+                    continue
+
+                # Normalize dimensions and cache
+                for orig_idx, text, emb in zip(indices, texts, embeddings):
+                    if emb is None:
+                        continue
+                    emb_array = np.asarray(emb, dtype=np.float16)
+                    if len(emb_array) != self.dimensions:
+                        if len(emb_array) < self.dimensions:
+                            emb_array = np.pad(emb_array, (0, self.dimensions - len(emb_array)))
+                        else:
+                            emb_array = emb_array[: self.dimensions]
+                    results[orig_idx] = emb_array
+                    self._put_to_cache(text, emb_array)
+
+        return results
+
+    async def get_node_embeddings(self, nodes: list[EmbNode], **kwargs) -> list[EmbNode]:
+        """Compute and assign embeddings for EmbNode objects."""
+        embeddings = await self.get_embeddings([n.text for n in nodes], **kwargs)
+        if len(embeddings) == len(nodes):
+            for node, vec in zip(nodes, embeddings):
+                if vec is not None:
+                    node.embedding = vec
+        return nodes
+
+    @abstractmethod
+    async def _get_embeddings(self, input_text: list[str], **kwargs) -> list[list[float] | None]:
+        """Get raw embeddings from the underlying provider."""
+
+    # -- Cache Operations --
+
+    def _get_from_cache(self, text: str) -> np.ndarray | None:
+        """Lookup text in LRU cache, promoting on hit."""
+        if not self.enable_cache:
+            return None
+        key = self._get_cache_key(text)
+        if key not in self._embedding_cache:
+            return None
+        self._embedding_cache.move_to_end(key)
+        return self._embedding_cache[key]
+
+    def _put_to_cache(self, text: str, embedding: np.ndarray) -> None:
+        """Insert into LRU cache, evicting oldest if full."""
+        if not self.enable_cache or self.max_cache_size <= 0 or len(embedding) != self.dimensions:
+            return
+        key = self._get_cache_key(text)
+        if len(self._embedding_cache) >= self.max_cache_size and key not in self._embedding_cache:
+            self._embedding_cache.popitem(last=False)
+        self._embedding_cache[key] = embedding
+        self._embedding_cache.move_to_end(key)
+
+    def _get_cache_key(self, text: str) -> str:
+        """Generate cache key from text, model name, and dimensions."""
+        return hashlib.sha256(f"{text}|{self.model_name}|{self.dimensions}".encode()).hexdigest()
+
+    # -- Cache Persistence --
+
+    async def load(self) -> None:
+        """Load cached embeddings from disk (npz format); replaces in-memory cache."""
+        self._embedding_cache.clear()
+        if not self.enable_cache or not self.cache_path.exists():
+            return
+
+        try:
+            data = np.load(self.cache_path)
+        except Exception:
+            self.logger.exception("Failed to load embedding cache, removing")
+            self.cache_path.unlink(missing_ok=True)
+            return
+
+        for key, emb in zip(data["keys"], data["embeddings"]):
+            if len(emb) != self.dimensions:
+                continue
+            if len(self._embedding_cache) >= self.max_cache_size:
+                break
+            self._embedding_cache[str(key)] = emb.astype(np.float16)
+        self.logger.info(f"Loaded {len(self._embedding_cache)} embeddings from {self.cache_path}")
+
+    async def dump(self) -> None:
+        """Persist in-memory cache to disk (npz format)."""
+        if not self.enable_cache or not self._embedding_cache:
+            return
+        self.cache_path.parent.mkdir(parents=True, exist_ok=True)
+        keys = list(self._embedding_cache.keys())
+        embeddings = np.stack(list(self._embedding_cache.values()))
+        try:
+            np.savez(self.cache_path, keys=np.array(keys, dtype=str), embeddings=embeddings)
+            self.logger.info(f"Saved {len(self._embedding_cache)} embeddings to {self.cache_path}")
+        except Exception:
+            self.logger.exception("Failed to save embedding cache")

reme4/components/embedding/openai_embedding_model.py

@@ -0,0 +1,52 @@
+"""OpenAI-compatible async embedding model."""
+
+from openai import AsyncOpenAI
+
+from .base_embedding_model import BaseEmbeddingModel
+from ..component_registry import R
+
+
+@R.register("openai")
+class OpenAIEmbeddingModel(BaseEmbeddingModel):
+    """Embedding model backed by any OpenAI-compatible API."""
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._client: AsyncOpenAI | None = None
+
+    async def _start(self) -> None:
+        """Initialize async OpenAI client."""
+        self._client = AsyncOpenAI(api_key=self.api_key, base_url=self.base_url, **self.kwargs)
+        await super()._start()
+
+    async def _close(self) -> None:
+        """Close the async OpenAI client."""
+        if self._client:
+            await self._client.close()
+            self._client = None
+        await super()._close()
+
+    async def _get_embeddings(self, input_text: list[str], **kwargs) -> list[list[float] | None]:
+        """Call the embeddings API and return results aligned to input order."""
+        if self._client is None:
+            raise RuntimeError("Client not initialized. Call _start() first.")
+
+        create_kwargs: dict = {"model": self.model_name, "input": input_text, **kwargs}
+        if self.pass_dimensions:
+            create_kwargs["dimensions"] = self.dimensions
+
+        completion = await self._client.embeddings.create(**create_kwargs)
+
+        # Map API results back to input order
+        result: list[list[float] | None] = [None] * len(input_text)
+        for emb in completion.data:
+            if 0 <= emb.index < len(input_text):
+                vec = emb.embedding or getattr(emb, "dense_embedding", None)
+                if vec is not None:
+                    result[emb.index] = list(vec)
+                else:
+                    self.logger.warning(f"Empty embedding at index {emb.index}")
+            else:
+                self.logger.warning(f"Index {emb.index} out of range for input length {len(input_text)}")
+
+        return result

reme4/components/file_graph/__init__.py

@@ -0,0 +1,8 @@
+"""File graph module."""
+
+from .base_file_graph import BaseFileGraph
+from .local_file_graph import LocalFileGraph
+from .neo4j_file_graph import Neo4jFileGraph
+from .nx_file_graph import NxFileGraph
+
+__all__ = ["BaseFileGraph", "LocalFileGraph", "Neo4jFileGraph", "NxFileGraph"]

reme4/components/file_graph/base_file_graph.py

@@ -0,0 +1,53 @@
+"""Abstract base for file-graph backends."""
+
+from abc import abstractmethod
+from pathlib import Path
+
+from ..base_component import BaseComponent
+from ...enumeration import ComponentEnum
+from ...schema import FileLink, FileNode
+
+
+class BaseFileGraph(BaseComponent):
+    """Abstract base for file-graph backends."""
+
+    component_type = ComponentEnum.FILE_GRAPH
+
+    def __init__(self, graph_name: str = "default", graph_version: str = "v1", **kwargs):
+        super().__init__(**kwargs)
+        self.graph_name: str = graph_name or self.name
+        self.graph_version: str = graph_version
+        self.graph_path: Path = self.working_metadata_path / self.component_type.value
+        self.graph_path.mkdir(parents=True, exist_ok=True)
+
+    # -- Node CRUD ---------------------------------------------------------
+
+    @abstractmethod
+    async def upsert_nodes(self, nodes: list[FileNode]) -> None:
+        """Insert or update nodes in the graph."""
+
+    @abstractmethod
+    async def delete_nodes(self, paths: list[str]) -> None:
+        """Delete nodes by path."""
+
+    @abstractmethod
+    async def get_nodes(self, paths: list[str] | None = None) -> list[FileNode]:
+        """Return nodes by paths; None = all real nodes; [] = []."""
+
+    @abstractmethod
+    async def rebuild_links(self) -> None:
+        """Rebuild all edges from each node's link payload."""
+
+    @abstractmethod
+    async def clear(self):
+        """Remove all nodes and edges."""
+
+    # -- Link access -------------------------------------------------------
+
+    @abstractmethod
+    async def get_outlinks(self, path: str) -> list[FileLink]:
+        """Return outgoing links for *path*."""
+
+    @abstractmethod
+    async def get_inlinks(self, path: str) -> list[FileLink]:
+        """Return incoming links for *path*."""

reme4/components/file_graph/local_file_graph.py

@@ -0,0 +1,138 @@
+"""Pure-Python file-graph backend (no external deps)."""
+
+from pathlib import Path
+
+from .base_file_graph import BaseFileGraph
+from ..component_registry import R
+from ...schema import FileLink, FileNode
+
+
+@R.register("local")
+class LocalFileGraph(BaseFileGraph):
+    """Dict-backed file graph; uses FileLink.target_path for adjacency."""
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._nodes: dict[str, FileNode] = {}
+        self._inverse: dict[str, set[str]] = {}  # target → {sources}
+        self._pending: dict[str, set[str]] = {}  # virtual target → {sources}
+        self._graph_file: Path = self.graph_path / f"{self.graph_name}_{self.graph_version}.jsonl"
+
+    # -- Lifecycle ---------------------------------------------------------
+
+    async def _start(self) -> None:
+        await super()._start()
+        await self.load()
+        await self.rebuild_links()
+
+    async def _close(self) -> None:
+        await self.dump()
+        await super()._close()
+
+    async def load(self) -> None:
+        """Load nodes from JSONL file into memory; keep current state on failure."""
+        if not self._graph_file.exists():
+            return
+        try:
+            with open(self._graph_file, "r", encoding="utf-8") as f:
+                self._nodes.update(
+                    (n.path, n) for line in f if line.strip() for n in [FileNode.model_validate_json(line)]
+                )
+            self.logger.info(f"Loaded {len(self._nodes)} nodes from {self._graph_file}")
+        except Exception as e:
+            self.logger.exception(f"Failed to load {self._graph_file}: {e}")
+
+    async def dump(self) -> None:
+        """Persist all nodes to JSONL via atomic rename."""
+        try:
+            tmp = self._graph_file.with_suffix(".tmp")
+            with open(tmp, "w", encoding="utf-8") as f:
+                f.writelines(f"{n.model_dump_json()}\n" for n in self._nodes.values())
+            tmp.replace(self._graph_file)
+            self.logger.info(f"Saved {len(self._nodes)} nodes to {self._graph_file}")
+        except Exception as e:
+            self.logger.exception(f"Failed to write {self._graph_file}: {e}")
+
+    # -- Edge bookkeeping --------------------------------------------------
+
+    def _add_edge(self, src: str, target: str) -> None:
+        """Register src→target; route to pending if target is virtual."""
+        bucket = self._inverse if target in self._nodes else self._pending
+        bucket.setdefault(target, set()).add(src)
+
+    def _remove_edge(self, src: str, target: str) -> None:
+        """Remove src→target from both inverse and pending buckets."""
+        for bucket in (self._inverse, self._pending):
+            srcs = bucket.get(target)
+            if srcs is None or src not in srcs:
+                continue
+            srcs.discard(src)
+            if not srcs:
+                del bucket[target]
+
+    # -- Node CRUD ---------------------------------------------------------
+
+    async def upsert_nodes(self, nodes: list[FileNode]) -> None:
+        for node in nodes:
+            path = node.path
+            old = self._nodes.get(path)
+            if old is not None:
+                for link in old.links:
+                    if link.target_path:
+                        self._remove_edge(path, link.target_path)
+            self._nodes[path] = node
+            for link in node.links:
+                if link.target_path:
+                    self._add_edge(path, link.target_path)
+            # Promote pending edges that now target a real node.
+            promoted = self._pending.pop(path, None)
+            if promoted:
+                self._inverse.setdefault(path, set()).update(promoted)
+
+    async def delete_nodes(self, paths: list[str]) -> None:
+        for path in paths:
+            node = self._nodes.pop(path, None)
+            if node is None:
+                continue
+            for link in node.links:
+                if link.target_path:
+                    self._remove_edge(path, link.target_path)
+            # Demote inbound edges to pending (sources still reference this path).
+            demoted = self._inverse.pop(path, None)
+            if demoted:
+                self._pending.setdefault(path, set()).update(demoted)
+
+    async def get_nodes(self, paths: list[str] | None = None) -> list[FileNode]:
+        if paths is None:
+            return list(self._nodes.values())
+        return [self._nodes[p] for p in paths if p in self._nodes]
+
+    async def rebuild_links(self) -> None:
+        """Rebuild inverse/pending indexes from all node link payloads."""
+        self._inverse.clear()
+        self._pending.clear()
+        for src, node in self._nodes.items():
+            for link in node.links:
+                if link.target_path:
+                    self._add_edge(src, link.target_path)
+
+    async def clear(self):
+        self._nodes.clear()
+        self._inverse.clear()
+        self._pending.clear()
+        self._graph_file.unlink(missing_ok=True)
+
+    # -- Link access -------------------------------------------------------
+
+    async def get_outlinks(self, path: str) -> list[FileLink]:
+        node = self._nodes.get(path)
+        if node is None:
+            return []
+        return [lnk for lnk in node.links if lnk.target_path and lnk.target_path in self._nodes]
+
+    async def get_inlinks(self, path: str) -> list[FileLink]:
+        if path not in self._nodes:
+            return []
+        return [
+            link for src in self._inverse.get(path, ()) for link in self._nodes[src].links if link.target_path == path
+        ]

reme4/components/file_graph/neo4j_file_graph.py

@@ -0,0 +1,450 @@
+"""Neo4j-backed file graph.
+
+Property-graph mapping:
+
+    Real node:    (:File {path, st_mtime, title, description, tags,
+                          chunk_ids, links_json, extra_json})
+    Virtual node: (:File {path})  — placeholder created when something
+                  links to a path that hasn't been upserted yet.
+
+    Edge:         (:File)-[:LINKS {idx, anchor, predicate}]->(:File)
+
+The ``links_json`` property doubles as the "is real" marker — its
+presence means the node was upserted with a payload; its absence
+means the node exists only because some edge points at it. This
+mirrors ``NxFileGraph`` exactly: ``upsert_nodes`` promotes virtuals
+in place, ``delete_nodes`` demotes back to virtual (or fully removes
+if nothing points here), and ``get_outlinks`` excludes edges into
+virtuals so the agent never sees dangling pointers.
+
+``path`` is the unique key (constraint enforced on ``_start``).
+Frontmatter goes into flat properties; arbitrary extras land in
+``extra_json``. The full ``FileLink[]`` payload is also stored as
+``links_json`` so ``rebuild_links`` can rebuild the relationship
+graph from per-node payloads after backend repair / migration.
+
+Adjacency policy: trusts ``FileLink.path`` directly — no internal
+wikilink resolution. The parser pipeline (with the external
+resolver) produces safe links where ``link.path`` is already a
+vault-relative target.
+
+Conditional dependency: the ``neo4j`` driver loads lazily; the
+import error fires at ``_start`` (boot), not at first call.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from .base_file_graph import BaseFileGraph
+from ..component_registry import R
+from ...schema import FileLink, FileNode
+from ...schema.file_node import FileFrontMatter
+
+
+_TYPED_FRONTMATTER_FIELDS = {"title", "description", "tags"}
+_LINK_FIELDS = {"source_path", "target_path", "target_anchor", "predicate"}
+
+# Properties that distinguish a "real" node from a virtual placeholder.
+# Listed for the demote query (delete_nodes) so we can REMOVE them all.
+_REAL_PROPS = (
+    "st_mtime",
+    "title",
+    "description",
+    "tags",
+    "chunk_ids",
+    "links_json",
+    "extra_json",
+)
+
+
+@R.register("neo4j")
+class Neo4jFileGraph(BaseFileGraph):
+    """Neo4j-backed file graph; trusts ``FileLink.path`` for adjacency.
+
+    Connection params (constructor kwargs):
+        uri:      bolt URL, e.g. ``bolt://localhost:7687``
+        user:     auth user (default ``neo4j``)
+        password: auth password
+        database: target db name (default ``neo4j``)
+    """
+
+    def __init__(
+        self,
+        uri: str = "bolt://localhost:7687",
+        user: str = "neo4j",
+        password: str = "neo4j",
+        database: str = "neo4j",
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self._uri: str = uri
+        self._user: str = user
+        self._password: str = password
+        self._database: str = database
+        self._driver = None
+
+    # -- Lifecycle ---------------------------------------------------------
+
+    async def _start(self) -> None:
+        await super()._start()
+        try:
+            from neo4j import AsyncGraphDatabase
+        except ImportError as e:
+            raise ImportError(
+                "Neo4jFileGraph requires the neo4j driver. Install with `pip install neo4j`.",
+            ) from e
+        self._driver = AsyncGraphDatabase.driver(
+            self._uri,
+            auth=(self._user, self._password),
+        )
+        async with self._session() as session:
+            await session.run(
+                "CREATE CONSTRAINT file_path_unique IF NOT EXISTS FOR (f:File) REQUIRE f.path IS UNIQUE",
+            )
+            real, virtual, edges = await self._counts(session)
+        self.logger.info(
+            f"Neo4jFileGraph '{self.graph_name}' connected at "
+            f"{self._uri}/{self._database}: "
+            f"{real} nodes, {edges} edges, {virtual} virtual",
+        )
+
+    async def _close(self) -> None:
+        if self._driver is not None:
+            await self._driver.close()
+            self._driver = None
+        await super()._close()
+
+    def _session(self):
+        assert self._driver is not None, "Neo4jFileGraph not started"
+        return self._driver.session(database=self._database)
+
+    @staticmethod
+    async def _counts(session) -> tuple[int, int, int]:
+        rec = await session.run(
+            """
+            MATCH (f:File)
+            WITH count(CASE WHEN f.links_json IS NOT NULL THEN 1 END) AS real,
+                 count(CASE WHEN f.links_json IS NULL THEN 1 END) AS virtual
+            OPTIONAL MATCH ()-[r:LINKS]->()
+            RETURN real, virtual, count(r) AS edges
+            """,
+        )
+        row = await rec.single()
+        if row is None:
+            return 0, 0, 0
+        return int(row["real"] or 0), int(row["virtual"] or 0), int(row["edges"] or 0)
+
+    # -- Node CRUD ---------------------------------------------------------
+
+    async def upsert_nodes(self, nodes: list[FileNode]) -> None:
+        """Upsert in one tx: SET props (promotes virtual to real), drop
+        existing outgoing edges, re-emit edges (auto-creating virtual
+        nodes for unindexed targets)."""
+        if not nodes:
+            return
+        payload = [
+            {
+                "path": node.path,
+                "props": self._node_props(node),
+                "links": [
+                    {
+                        "idx": i,
+                        "anchor": link.target_anchor,
+                        "predicate": link.predicate,
+                        "target": link.target_path,
+                    }
+                    for i, link in enumerate(node.links)
+                    if link.target_path
+                ],
+            }
+            for node in nodes
+        ]
+        async with self._session() as session:
+            await session.execute_write(self._upsert_nodes_tx, payload)
+
+    @staticmethod
+    async def _upsert_nodes_tx(tx, payload):
+        # 1. Upsert node props (promotes virtual → real where necessary).
+        await tx.run(
+            """
+            UNWIND $items AS n
+            MERGE (f:File {path: n.path})
+            SET f += n.props
+            """,
+            items=payload,
+        )
+        # 2. Drop existing outgoing edges from these sources.
+        await tx.run(
+            """
+            UNWIND $paths AS p
+            MATCH (f:File {path: p})-[r:LINKS]->()
+            DELETE r
+            """,
+            paths=[item["path"] for item in payload],
+        )
+        # 3. Re-emit edges; MERGE on target auto-creates virtual nodes
+        # for unindexed targets.
+        await tx.run(
+            """
+            UNWIND $items AS n
+            MATCH (s:File {path: n.path})
+            UNWIND n.links AS link
+            MERGE (t:File {path: link.target})
+            MERGE (s)-[r:LINKS {idx: link.idx}]->(t)
+            SET r.anchor = link.anchor, r.predicate = link.predicate
+            """,
+            items=payload,
+        )
+
+    async def delete_nodes(self, paths: list[str]) -> None:
+        """Demote real → virtual to preserve inbound visibility; fully
+        remove the (now-virtual) node only if no edge points at it."""
+        if not paths:
+            return
+        async with self._session() as session:
+            await session.execute_write(self._delete_nodes_tx, list(paths))
+
+    @staticmethod
+    async def _delete_nodes_tx(tx, paths):
+        # 1. Drop outgoing edges, then strip "real" properties (demote).
+        # Building the REMOVE clause from _REAL_PROPS keeps the list of
+        # properties in one place (top of module).
+        remove_clause = ", ".join(f"f.{name}" for name in _REAL_PROPS)
+        await tx.run(
+            f"""
+            UNWIND $paths AS p
+            MATCH (f:File {{path: p}})
+            OPTIONAL MATCH (f)-[r:LINKS]->()
+            DELETE r
+            WITH DISTINCT f
+            REMOVE {remove_clause}
+            """,
+            paths=paths,
+        )
+        # 2. Garbage-collect: drop the virtual node entirely if nothing
+        # points at it anymore.
+        await tx.run(
+            """
+            UNWIND $paths AS p
+            MATCH (f:File {path: p})
+            WHERE f.links_json IS NULL AND NOT (f)<-[:LINKS]-()
+            DELETE f
+            """,
+            paths=paths,
+        )
+
+    async def get_nodes(self, paths: list[str] | None = None) -> list[FileNode]:
+        """Return real nodes (virtual placeholders filtered).
+
+        ``paths=None`` streams every real node ordered by path. An
+        explicit ``[]`` returns ``[]`` without hitting the database.
+        """
+        if paths is not None and not paths:
+            return []
+        async with self._session() as session:
+            if paths is None:
+                rec = await session.run(
+                    """
+                    MATCH (f:File)
+                    WHERE f.links_json IS NOT NULL
+                    RETURN f
+                    ORDER BY f.path ASC
+                    """,
+                )
+            else:
+                rec = await session.run(
+                    """
+                    UNWIND $paths AS p
+                    MATCH (f:File {path: p})
+                    WHERE f.links_json IS NOT NULL
+                    RETURN f
+                    """,
+                    paths=list(paths),
+                )
+            rows = [row["f"] async for row in rec]
+        return [self._row_to_node(row) for row in rows]
+
+    async def rebuild_links(self) -> None:
+        """Defensive full rebuild from each real node's ``links_json``.
+
+        Three steps in one tx: drop all LINKS edges; drop all virtual
+        nodes; re-emit edges from per-node link payloads (re-creating
+        virtual targets as needed). Useful after manual repair or
+        schema migration.
+        """
+        async with self._session() as session:
+            rec = await session.run(
+                """
+                MATCH (f:File)
+                WHERE f.links_json IS NOT NULL
+                RETURN f.path AS p, f.links_json AS l
+                """,
+            )
+            rows = [dict(r) async for r in rec]
+
+        payload: list[dict] = []
+        for row in rows:
+            try:
+                links = json.loads(row.get("l") or "[]")
+            except json.JSONDecodeError:
+                continue
+            items = [
+                {
+                    "idx": i,
+                    "anchor": link.get("target_anchor"),
+                    "predicate": link.get("predicate"),
+                    "target": link.get("target_path"),
+                }
+                for i, link in enumerate(links)
+                if isinstance(link, dict) and link.get("target_path")
+            ]
+            payload.append({"path": row["p"], "links": items})
+
+        async with self._session() as session:
+            await session.execute_write(self._rebuild_links_tx, payload)
+
+    @staticmethod
+    async def _rebuild_links_tx(tx, payload):
+        # 1. Wipe all edges and all virtual nodes.
+        await tx.run("MATCH ()-[r:LINKS]->() DELETE r")
+        await tx.run("MATCH (f:File) WHERE f.links_json IS NULL DELETE f")
+        if not payload:
+            return
+        # 2. Re-emit edges; virtual targets reappear via MERGE.
+        await tx.run(
+            """
+            UNWIND $items AS n
+            MATCH (s:File {path: n.path})
+            UNWIND n.links AS link
+            MERGE (t:File {path: link.target})
+            MERGE (s)-[r:LINKS {idx: link.idx}]->(t)
+            SET r.anchor = link.anchor, r.predicate = link.predicate
+            """,
+            items=payload,
+        )
+
+    async def clear(self):
+        """Remove every node and edge in the configured database."""
+        async with self._session() as session:
+            await session.run("MATCH (f:File) DETACH DELETE f")
+
+    # -- Link access -------------------------------------------------------
+
+    async def get_outlinks(self, path: str) -> list[FileLink]:
+        """Outgoing links from ``path``. Source must be real; targets
+        into virtual nodes are excluded so dangling refs are invisible."""
+        async with self._session() as session:
+            rec = await session.run(
+                """
+                MATCH (s:File {path: $path})
+                WHERE s.links_json IS NOT NULL
+                MATCH (s)-[r:LINKS]->(t:File)
+                WHERE t.links_json IS NOT NULL
+                RETURN t.path AS target, r.anchor AS anchor,
+                       r.predicate AS predicate, r.idx AS idx
+                ORDER BY r.idx ASC
+                """,
+                path=path,
+            )
+            rows = [dict(row) async for row in rec]
+        return [
+            FileLink(
+                source_path=path,
+                target_path=row["target"],
+                target_anchor=row.get("anchor"),
+                predicate=row.get("predicate"),
+            )
+            for row in rows
+        ]
+
+    async def get_inlinks(self, path: str) -> list[FileLink]:
+        """Incoming links to ``path`` (must be real). Sources are always
+        real because virtual nodes never have outgoing edges."""
+        async with self._session() as session:
+            rec = await session.run(
+                """
+                MATCH (t:File {path: $path})
+                WHERE t.links_json IS NOT NULL
+                MATCH (s:File)-[r:LINKS]->(t)
+                RETURN r.anchor AS anchor, r.predicate AS predicate,
+                       r.idx AS idx, s.path AS source
+                ORDER BY s.path ASC, r.idx ASC
+                """,
+                path=path,
+            )
+            rows = [dict(row) async for row in rec]
+        return [
+            FileLink(
+                source_path=row["source"],
+                target_path=path,
+                target_anchor=row.get("anchor"),
+                predicate=row.get("predicate"),
+            )
+            for row in rows
+        ]
+
+    # -- Internal: row ↔ schema marshaling ---------------------------------
+
+    @staticmethod
+    def _node_props(node: FileNode) -> dict[str, Any]:
+        fm = node.front_matter
+        extras = dict(fm.__pydantic_extra__ or {})
+        return {
+            "path": node.path,
+            "st_mtime": float(node.st_mtime),
+            "title": fm.title or "",
+            "description": fm.description or "",
+            "tags": list(fm.tags or []),
+            "chunk_ids": list(node.chunk_ids or []),
+            "links_json": json.dumps(
+                [link.model_dump(exclude_none=True) for link in node.links],
+                ensure_ascii=False,
+            ),
+            "extra_json": json.dumps(extras, ensure_ascii=False, sort_keys=True),
+        }
+
+    @staticmethod
+    def _row_to_node(row) -> FileNode:
+        d = dict(row)
+        try:
+            extras = json.loads(d.get("extra_json") or "{}")
+        except json.JSONDecodeError:
+            extras = {}
+        try:
+            links_raw = json.loads(d.get("links_json") or "[]")
+        except json.JSONDecodeError:
+            links_raw = []
+        links: list[FileLink] = []
+        for link in links_raw:
+            if not isinstance(link, dict):
+                continue
+            # Defensive: strip any keys the schema doesn't recognise
+            # (e.g. legacy fields from prior schema versions).
+            clean = {k: v for k, v in link.items() if k in _LINK_FIELDS}
+            # Ensure source_path is populated — older payloads (or
+            # links written before the schema split) only carry the
+            # target side; default to the owning node's path.
+            clean.setdefault("source_path", d["path"])
+            if not clean.get("target_path"):
+                continue
+            try:
+                links.append(FileLink(**clean))
+            except Exception:
+                continue
+        fm_kwargs: dict[str, Any] = {
+            "title": d.get("title", "") or "",
+            "description": d.get("description", "") or "",
+            "tags": d.get("tags") or None,
+        }
+        fm_kwargs.update(
+            {k: v for k, v in extras.items() if k not in _TYPED_FRONTMATTER_FIELDS},
+        )
+        return FileNode(
+            path=d["path"],
+            st_mtime=float(d.get("st_mtime", 0.0)),
+            links=links,
+            chunk_ids=[str(c) for c in (d.get("chunk_ids") or [])],
+            front_matter=FileFrontMatter(**fm_kwargs),
+        )

reme4/components/file_graph/nx_file_graph.py

@@ -0,0 +1,122 @@
+"""Networkx file-graph backend."""
+
+import pickle
+from pathlib import Path
+
+try:
+    import networkx as nx
+except ImportError:
+    nx = None
+
+from .base_file_graph import BaseFileGraph
+from ..component_registry import R
+from ...schema import FileLink, FileNode
+
+
+@R.register("nx")
+class NxFileGraph(BaseFileGraph):
+    """Networkx-backed file graph; uses FileLink.target_path for adjacency."""
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        if nx is None:
+            raise ImportError("NxFileGraph requires networkx — pip install networkx")
+        self._graph: nx.MultiDiGraph = nx.MultiDiGraph()
+        self._graph_file: Path = self.graph_path / f"{self.graph_name}_{self.graph_version}.pkl"
+
+    # -- Lifecycle ---------------------------------------------------------
+
+    async def _start(self) -> None:
+        await super()._start()
+        await self.load()
+
+    async def _close(self) -> None:
+        await self.dump()
+        await super()._close()
+
+    async def load(self) -> None:
+        """Load graph from pickle file; keep current graph on failure."""
+        if not self._graph_file.exists():
+            return
+        try:
+            with open(self._graph_file, "rb") as f:
+                self._graph = pickle.load(f)
+            n_real = sum(1 for _, d in self._graph.nodes(data=True) if "node" in d)
+            self.logger.info(f"Loaded {n_real} nodes from {self._graph_file}")
+        except Exception as e:
+            self.logger.exception(f"Failed to load {self._graph_file}: {e}")
+
+    async def dump(self) -> None:
+        """Persist graph to pickle via atomic rename."""
+        try:
+            tmp = self._graph_file.with_suffix(".tmp")
+            with open(tmp, "wb") as f:
+                pickle.dump(self._graph, f, protocol=pickle.HIGHEST_PROTOCOL)
+            tmp.replace(self._graph_file)
+            n_real = sum(1 for _, d in self._graph.nodes(data=True) if "node" in d)
+            self.logger.info(f"Saved {n_real} nodes to {self._graph_file}")
+        except Exception as e:
+            self.logger.exception(f"Failed to write {self._graph_file}: {e}")
+
+    # -- Node CRUD ---------------------------------------------------------
+
+    async def upsert_nodes(self, nodes: list[FileNode]) -> None:
+        for node in nodes:
+            path = node.path
+            if self._graph.has_node(path):
+                # Drop outgoing edges; inbound stay intact.
+                self._graph.remove_edges_from(list(self._graph.out_edges(path, keys=True)))
+            self._graph.add_node(path, node=node)  # promotes virtual node if present
+            # Missing targets become attr-less virtual nodes.
+            self._graph.add_edges_from((path, lnk.target_path, {"link": lnk}) for lnk in node.links if lnk.target_path)
+
+    async def delete_nodes(self, paths: list[str]) -> None:
+        for path in paths:
+            if not self._graph.has_node(path):
+                continue
+            self._graph.remove_edges_from(list(self._graph.out_edges(path, keys=True)))
+            # Demote to virtual: keep inbound edges, drop node payload.
+            self._graph.nodes[path].pop("node", None)
+            if self._graph.in_degree(path) == 0:
+                self._graph.remove_node(path)  # remove orphan virtual node
+
+    async def get_nodes(self, paths: list[str] | None = None) -> list[FileNode]:
+        nodes_view = self._graph.nodes
+        if paths is None:
+            return [d["node"] for _, d in nodes_view(data=True) if "node" in d]
+        return [nodes_view[path]["node"] for path in paths if path in nodes_view and "node" in nodes_view[path]]
+
+    async def rebuild_links(self) -> None:
+        """Rebuild all edges from real node payloads; drop virtual nodes."""
+        self._graph.remove_edges_from(list(self._graph.edges(keys=True)))
+        virtual = [n for n, d in self._graph.nodes(data=True) if "node" not in d]
+        self._graph.remove_nodes_from(virtual)
+        self._graph.add_edges_from(
+            (path, lnk.target_path, {"link": lnk})
+            for path, data in self._graph.nodes(data=True)
+            for lnk in data["node"].links
+            if lnk.target_path
+        )
+
+    async def clear(self):
+        """Remove all nodes and edges, and remove persisted file."""
+        self._graph.clear()
+        self._graph_file.unlink(missing_ok=True)
+
+    # -- Link access -------------------------------------------------------
+
+    async def get_outlinks(self, path: str) -> list[FileLink]:
+        nodes_view = self._graph.nodes
+        if path not in nodes_view or "node" not in nodes_view[path]:
+            return []
+        return [
+            d["link"]
+            for _, target, d in self._graph.out_edges(path, data=True)
+            if "link" in d and "node" in nodes_view[target]
+        ]
+
+    async def get_inlinks(self, path: str) -> list[FileLink]:
+        nodes_view = self._graph.nodes
+        if path not in nodes_view or "node" not in nodes_view[path]:
+            return []
+        return [d["link"] for _, _, d in self._graph.in_edges(path, data=True) if "link" in d]

reme4/components/file_parser/__init__.py

@@ -0,0 +1,8 @@
+"""File parser components."""
+
+from .bare_file_parser import BareFileParser
+from .base_file_parser import BaseFileParser
+from .default_file_parser import DefaultFileParser
+from .linked_file_parser import LinkedFileParser
+
+__all__ = ["BareFileParser", "BaseFileParser", "DefaultFileParser", "LinkedFileParser"]

reme4/components/file_parser/bare_file_parser.py

@@ -0,0 +1,22 @@
+"""Stat-only parser for attachment/binary files."""
+
+from pathlib import Path
+
+from .base_file_parser import BaseFileParser
+from ..component_registry import R
+from ...schema import FileChunk, FileNode
+
+
+@R.register("bare")
+class BareFileParser(BaseFileParser):
+    """Stat-only parser for attachment/binary files.
+
+    No content read, no chunking, no link extraction. The resulting FileNode
+    has empty links and chunk_ids; front_matter carries mime and size so
+    retrieval can filter by file type without reopening the file.
+    """
+
+    async def parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]:
+        file_path = Path(path)
+        stat = file_path.stat()
+        return FileNode(path=self._get_relative_path(path), st_mtime=stat.st_mtime, links=[], chunk_ids=[]), []

reme4/components/file_parser/base_file_parser.py

@@ -0,0 +1,30 @@
+"""Abstract base for file parsers."""
+
+from abc import abstractmethod
+from pathlib import Path
+
+from ..base_component import BaseComponent
+from ...enumeration import ComponentEnum
+from ...schema import FileChunk, FileNode
+
+
+class BaseFileParser(BaseComponent):
+    """Abstract base for file parsers. Subclasses implement `parse`."""
+
+    component_type = ComponentEnum.FILE_PARSER
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.working_dir = self.app_context.app_config.working_dir if self.app_context else ""
+
+    def _get_relative_path(self, path: str | Path) -> str:
+        """Return path relative to working_dir, or absolute path if outside."""
+        file_path = Path(path).absolute()
+        try:
+            return str(file_path.relative_to(Path(self.working_dir).absolute()))
+        except ValueError:
+            return str(file_path)
+
+    @abstractmethod
+    async def parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]:
+        """Parse a file into (node, chunks)."""

reme4/components/file_parser/default_file_parser.py

@@ -0,0 +1,123 @@
+"""Default file parser with byte-based overlapping chunking."""
+
+import re
+from bisect import bisect_right
+from pathlib import Path
+
+import aiofiles
+import yaml
+
+from .base_file_parser import BaseFileParser
+from ..component_registry import R
+from ...schema import FileChunk, FileFrontMatter, FileLink, FileNode
+
+# Single-pass wikilink + optional Dataview predicate.
+# Covers: [[X]] / [[X#h]] / [[X|alias]] / pred:: [[X]] / [pred:: [[X]]]
+# - predicate group: optional leading '[' (Dataview inline-bracket form), an identifier,
+#   then '::' — the whole prefix is non-capturing-optional so bare wikilinks still match.
+# - target / anchor: target stops before '#', '|', '[', ']'; anchor stops before '|', '[', ']'.
+# - alias '|...': consumed but not captured (we don't need display text).
+_LINK_RE = re.compile(
+    r"(?:\[?\s*(?P<predicate>[A-Za-z][\w-]*)\s*::\s*)?"
+    r"\[\[\s*(?P<target>[^\[\]|#]+?)"
+    r"(?:#(?P<anchor>[^\[\]|]+?))?"
+    r"\s*(?:\|[^\[\]]*?)?\s*\]\]",
+)
+
+
+@R.register("default")
+class DefaultFileParser(BaseFileParser):
+    """Parser that splits files into byte-based overlapping chunks."""
+
+    def __init__(self, encoding: str = "utf-8", chunk_byte_size: int = 10000, overlap_byte_size: int = 100, **kwargs):
+        super().__init__(**kwargs)
+        self.encoding = encoding
+        self.chunk_byte_size = max(100, chunk_byte_size)
+        self.overlap_byte_size = max(4, overlap_byte_size)
+
+    @staticmethod
+    def parse_links(content: str, source_path: str) -> list[FileLink]:
+        """Extract wikilinks with optional Dataview predicate as outgoing FileLinks."""
+        links: list[FileLink] = []
+        for m in _LINK_RE.finditer(content):
+            target = m["target"].strip()
+            if not target:
+                continue
+            anchor = m["anchor"]
+            links.append(
+                FileLink(
+                    source_path=source_path,
+                    target_path=target,
+                    target_anchor=anchor.strip() if anchor else None,
+                    predicate=m["predicate"],
+                ),
+            )
+        return links
+
+    @staticmethod
+    def _parse_front_matter(text: str) -> tuple[FileFrontMatter, str]:
+        """Parse YAML front matter delimited by ---, return (front_matter, remaining)."""
+        if not text.startswith("---"):
+            return FileFrontMatter(), text
+        end_idx = text.find("\n---", 3)
+        if end_idx == -1:
+            return FileFrontMatter(), text
+        try:
+            data = yaml.safe_load(text[3:end_idx].strip()) or {}
+            front_matter = FileFrontMatter(**(data if isinstance(data, dict) else {}))
+        except yaml.YAMLError:
+            front_matter = FileFrontMatter()
+        return front_matter, text[end_idx + 4 :].lstrip("\n")
+
+    async def parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]:
+        file_path = Path(path)
+        stat = file_path.stat()
+        rel_path = self._get_relative_path(path)
+
+        async with aiofiles.open(file_path, encoding=self.encoding) as f:
+            text = await f.read()
+
+        if not text:
+            return FileNode(path=rel_path, st_mtime=stat.st_mtime), []
+
+        front_matter, content = self._parse_front_matter(text)
+        if not content:
+            return FileNode(path=rel_path, st_mtime=stat.st_mtime, front_matter=front_matter), []
+
+        links = self.parse_links(content, rel_path)
+        chunks = self._chunk_content(content, rel_path)
+        chunk_ids = [c.id for c in chunks]
+        return (
+            FileNode(
+                path=rel_path,
+                st_mtime=stat.st_mtime,
+                front_matter=front_matter,
+                links=links,
+                chunk_ids=chunk_ids,
+            ),
+            chunks,
+        )
+
+    def _chunk_content(self, content: str, rel_path: str) -> list[FileChunk]:
+        """Split content into overlapping byte-range chunks with line numbers."""
+        content_bytes = content.encode(self.encoding)
+        newline_positions = [i for i, b in enumerate(content_bytes) if b == ord("\n")]
+        chunks: list[FileChunk] = []
+        step = self.chunk_byte_size - self.overlap_byte_size
+        start = 0
+
+        while start < len(content_bytes):
+            end = min(start + self.chunk_byte_size, len(content_bytes))
+            chunk_text = content_bytes[start:end].decode(self.encoding, errors="ignore")
+            start_line = bisect_right(newline_positions, start - 1) + 1
+            end_line = bisect_right(newline_positions, end - 1) + 1
+            if content_bytes[end - 1] == ord("\n"):
+                end_line -= 1
+            chunks.append(
+                FileChunk(path=rel_path, start_line=start_line, end_line=end_line, text=chunk_text).set_hash_id(),
+            )
+            if end >= len(content_bytes):
+                break
+            start += step
+
+        return chunks

reme4/components/file_parser/linked_file_parser.py

@@ -0,0 +1,729 @@
+"""Markdown file parser — frontmatter + wikilink graph + AST tree chunks.
+
+Each chunk carries the **complete heading skeleton** of the document
+with its content inlined under the section that owns it; other sections
+appear as bare headings so the reader always sees a full document map.
+
+Pipeline: build mistletoe AST → ``MdNode`` tree (sections nest by
+heading level) → recursive chunk (try whole subtree; on overflow walk
+children — body siblings pack as a run, subsections recurse). Leaf
+blocks (table / code / list / paragraph) split on internal boundaries
+and each piece is annotated ``[Part X/N]``. Wikilinks in the body are
+extracted as graph edges, with optional Dataview-style typed predicates
+(line-level ``predicate:: [[X]]`` or inline-bracketed ``[predicate:: [[X]]]``).
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import frontmatter
+
+
+from .base_file_parser import BaseFileParser
+from ..component_registry import R
+from ..file_graph import BaseFileGraph
+from ...enumeration import ComponentEnum
+from ...schema import (
+    FileChunk,
+    FileLink,
+    FileFrontMatter,
+    FileNode,
+)
+
+
+# -- Wikilink resolution --------------------------------------------------
+#
+# Wikilinks are a markdown user-facing convention: ``[[Alice]]`` should
+# resolve to ``topics/Alice/Alice.md`` (or wherever the file lives).
+# This short-form / implicit-``.md`` / folder-note resolution lives here
+# at the markdown boundary rather than as a generic utility — file-IO
+# steps require full vault-relative paths and never use these helpers.
+
+
+def _complete_md(target: str) -> str:
+    """Apply implicit ``.md`` rule for wikilink targets."""
+    if not target:
+        return target
+    last = target.rsplit("/", 1)[-1]
+    return target if "." in last else target + ".md"
+
+
+def _filter_folder_note(target: str, paths: list[str]) -> list[str]:
+    """Apply folder-note rule: when both ``X.md`` and ``X/X.md`` exist,
+    prefer ``X/X.md``. Sorted for determinism.
+    """
+    if not paths:
+        return []
+    stem = Path(target).stem
+    folder_hits = sorted(p for p in paths if Path(p).parent.name == stem)
+    return folder_hits or sorted(paths)
+
+
+async def _resolve_wikilink(graph: BaseFileGraph, target: str) -> list[str]:
+    """Resolve a wikilink target to vault-relative path(s).
+
+    Returns:
+        ``[path]`` for an unambiguous match,
+        ``[path, path, ...]`` for short-form ambiguity (caller may
+        fan out one FileLink per candidate), or
+        ``[]`` when nothing matches (dangling — caller drops the link).
+    """
+    if not target:
+        return []
+    target = _complete_md(target)
+    if "/" in target:
+        nodes = await graph.get_nodes([target])
+        return [target] if nodes else []
+    matches = [n.path for n in await graph.get_nodes() if Path(n.path).name == target]
+    return _filter_folder_note(target, matches)
+
+
+# -- Wikilink extraction --------------------------------------------------
+
+
+_WIKILINK_RE = re.compile(
+    r"""
+    (?:!)?
+    \[\[
+        (?P<target>[^\]\|\#\n]+?)
+        (?:\#(?P<anchor>[^\]\|\n]+))?
+        (?:\|[^\]\n]+)?
+    \]\]
+    """,
+    re.VERBOSE,
+)
+
+_DATAVIEW_LINE_RE = re.compile(
+    r"^[ \t]*(?:[-*+][ \t]+)?(?P<predicate>[A-Za-z][A-Za-z0-9_]*)\s*::\s*(?P<value>.+?)\s*$",
+    re.MULTILINE,
+)
+
+_INLINE_FIELD_OPEN_RE = re.compile(r"\[(?P<predicate>[A-Za-z][A-Za-z0-9_]*)\s*::\s*")
+
+
+def _iter_inline_fields(text: str) -> list[tuple[int, int, str]]:
+    """Find inline-bracketed ``[predicate:: …]`` field spans by depth scan."""
+    out: list[tuple[int, int, str]] = []
+    for m in _INLINE_FIELD_OPEN_RE.finditer(text):
+        depth = 1
+        i = m.end()
+        n = len(text)
+        while i < n:
+            c = text[i]
+            if c == "\n":
+                break
+            if c == "[":
+                depth += 1
+            elif c == "]":
+                depth -= 1
+                if depth == 0:
+                    out.append((m.start(), i + 1, m.group("predicate")))
+                    break
+            i += 1
+    return out
+
+
+def _predicate_for(
+    text: str,
+    pos: int,
+    inline_spans: list[tuple[int, int, str]],
+) -> str | None:
+    """Resolve the predicate governing a wikilink at offset ``pos``.
+
+    Precedence: inline-bracketed > line-level Dataview > none.
+    """
+    for field_start, field_end, predicate in inline_spans:
+        if field_start <= pos < field_end:
+            return predicate
+    line_start = text.rfind("\n", 0, pos) + 1
+    line_end = text.find("\n", pos)
+    if line_end == -1:
+        line_end = len(text)
+    m = _DATAVIEW_LINE_RE.match(text[line_start:line_end])
+    if m and line_start + m.start("value") <= pos:
+        return m.group("predicate")
+    return None
+
+
+async def _extract_links(
+    graph: BaseFileGraph,
+    text: str,
+    source_path: str,
+) -> list[FileLink]:
+    """Find every wikilink in ``text``, resolve targets, emit FileLinks.
+
+    Short-path ambiguity **expands** into one FileLink per candidate so
+    the body's wikilink is recorded against every plausible target.
+    Dangling targets are dropped. Results are deduped by
+    ``(target_path, predicate, target_anchor)`` preserving order.
+    """
+    if not text:
+        return []
+    inline_spans = _iter_inline_fields(text)
+    out: list[FileLink] = []
+    seen: set[tuple] = set()
+    for wm in _WIKILINK_RE.finditer(text):
+        target = wm.group("target").strip()
+        if not target:
+            continue
+        anchor_raw = wm.group("anchor")
+        anchor = anchor_raw.strip() if anchor_raw else None
+        predicate = _predicate_for(text, wm.start(), inline_spans)
+        resolved_paths = await _resolve_wikilink(graph, target)
+        if not resolved_paths:
+            continue
+        for resolved in resolved_paths:
+            key = (resolved, predicate, anchor)
+            if key in seen:
+                continue
+            seen.add(key)
+            out.append(
+                FileLink(
+                    source_path=source_path,
+                    target_path=resolved,
+                    target_anchor=anchor,
+                    predicate=predicate,
+                ),
+            )
+    return out
+
+
+# -- AST node + helpers ---------------------------------------------------
+
+
+@dataclass
+class MdNode:
+    """``root`` / ``section`` (heading + children until equal-or-shallower
+    heading) / ``body`` (one mistletoe block; ``block`` keeps the original).
+
+    ``text`` is the rendered subtree (own heading excluded for sections).
+    ``desc_toc`` caches the section-only DFS outline of descendants
+    (own heading excluded), used as the TOC suffix when emitting chunks
+    inside a section. Line ranges span the full subtree.
+    """
+
+    kind: str  # "root" | "section" | "body"
+    heading: str | None = None
+    level: int = 0
+    children: list["MdNode"] = field(default_factory=list)
+    block: Any = None
+    text: str = ""
+    start_line: int = 0
+    end_line: int = 0
+    desc_toc: str = ""
+
+
+def _heading_text(node: Any, renderer) -> str:
+    """Heading text without `#` markers (for outline)."""
+    rendered = renderer.render(node).rstrip("\n")
+    if rendered.startswith("#"):
+        return rendered.lstrip("#").strip()
+    return rendered.split("\n", 1)[0].strip()
+
+
+def _finalize(n: MdNode) -> None:
+    """Bottom-up pass: propagate line ranges, populate ``n.text`` (rendered
+    subtree, own heading excluded for sections) and ``n.desc_toc`` (DFS
+    section outline of descendants)."""
+    parts: list[str] = []
+    desc_lines: list[str] = []
+    for c in n.children:
+        _finalize(c)
+        if c.kind == "section":
+            heading = f"{'#' * c.level} {c.heading or ''}"
+            parts.append(f"{heading}\n\n{c.text}" if c.text else heading)
+            desc_lines.append(f"{heading}\n\n{c.desc_toc}" if c.desc_toc else heading)
+        elif c.text:
+            parts.append(c.text)
+    if n.children:
+        first = n.children[0].start_line
+        n.start_line = min(n.start_line, first) if n.start_line else first
+        n.end_line = max(c.end_line for c in n.children)
+    elif n.end_line < n.start_line:
+        n.end_line = n.start_line
+    if n.kind != "body":
+        n.text = "\n\n".join(parts)
+    n.desc_toc = "\n\n".join(desc_lines)
+
+
+def _toc_join(*parts: str) -> str:
+    """Concatenate TOC fragments with ``\\n\\n``, skipping empty ones."""
+    return "\n\n".join(p for p in parts if p)
+
+
+def _subtree_toc(n: MdNode) -> str:
+    """Section's heading + descendants TOC — its contribution to a parent's
+    ``desc_toc``. For root (no own heading) this is just ``desc_toc``."""
+    if n.kind != "section" or n.heading is None:
+        return n.desc_toc
+    heading = f"{'#' * n.level} {n.heading}"
+    return f"{heading}\n\n{n.desc_toc}" if n.desc_toc else heading
+
+
+# -- Parser ---------------------------------------------------------------
+
+
+@R.register("md")
+class LinkedFileParser(BaseFileParser):
+    """Markdown parser: frontmatter + wikilink edges + full-skeleton chunks."""
+
+    def __init__(
+        self,
+        encoding: str = "utf-8",
+        chunk_chars: int = 2000,
+        embed_toc: bool = True,
+        file_graph: str = "default",
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.encoding = encoding
+        self.chunk_chars = max(100, chunk_chars)
+        self.embed_toc = embed_toc
+        self._file_graph_name: str = file_graph
+
+    def _resolve_file_graph(self) -> BaseFileGraph | None:
+        """Lazily fetch the configured file_graph from app_context.
+
+        Lazy (rather than ``_start``) so the parser doesn't impose a
+        component start-order constraint, and so tests can construct
+        the parser without a graph wired up.
+        """
+        if self.app_context is None:
+            return None
+        graphs = self.app_context.components.get(ComponentEnum.FILE_GRAPH, {})
+        graph = graphs.get(self._file_graph_name)
+        if graph is None:
+            return None
+        if not isinstance(graph, BaseFileGraph):
+            raise TypeError(
+                f"Expected BaseFileGraph, got {type(graph).__name__}",
+            )
+        return graph
+
+    async def parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]:
+        from mistletoe.markdown_renderer import MarkdownRenderer
+        from mistletoe.block_token import Document
+
+        file_path = Path(path)
+        rel_path = self._get_relative_path(path)
+        post = frontmatter.loads(file_path.read_text(encoding=self.encoding))
+
+        chunks: list[FileChunk] = []
+        if post.content and post.content.strip():
+            with MarkdownRenderer() as renderer:
+                tree = self._build_tree(Document(post.content), renderer)
+                chunks = self._chunk_node(tree, "", "", rel_path, renderer)
+
+        links: list[FileLink] = []
+        graph = self._resolve_file_graph()
+        if graph is not None:
+            links = await _extract_links(graph, post.content, rel_path)
+
+        node = FileNode(
+            path=rel_path,
+            st_mtime=file_path.stat().st_mtime,
+            chunk_ids=[chunk.id for chunk in chunks],
+            links=links,
+            front_matter=FileFrontMatter(**dict(post.metadata)),
+        )
+        return node, chunks
+
+    def _build_tree(self, doc: Any, renderer) -> MdNode:
+        """Heading-level stack folds mistletoe's flat children into nested
+        sections; non-headings attach as ``body`` to the current section
+        (or root before the first heading)."""
+        from mistletoe.markdown_renderer import BlankLine
+        from mistletoe.block_token import (
+            Heading,
+            SetextHeading,
+        )
+
+        root = MdNode(kind="root", start_line=1, end_line=1)
+        stack: list[MdNode] = [root]
+        for child in doc.children or []:
+            if isinstance(child, BlankLine):
+                continue
+            line = getattr(child, "line_number", None) or stack[-1].start_line
+            if isinstance(child, (Heading, SetextHeading)):
+                level = max(1, getattr(child, "level", 1))
+                while len(stack) > 1 and stack[-1].level >= level:
+                    stack.pop()
+                sec = MdNode(
+                    kind="section",
+                    heading=_heading_text(child, renderer),
+                    level=level,
+                    start_line=line,
+                )
+                stack[-1].children.append(sec)
+                stack.append(sec)
+                continue
+            rendered = renderer.render(child).rstrip("\n")
+            if not rendered:
+                continue
+            stack[-1].children.append(
+                MdNode(
+                    kind="body",
+                    block=child,
+                    text=rendered,
+                    start_line=line,
+                    end_line=line + rendered.count("\n"),
+                ),
+            )
+        _finalize(root)
+        return root
+
+    # -- Recursive chunker ------------------------------------------------
+
+    def _chunk_node(
+        self,
+        node: MdNode,
+        before: str,
+        after: str,
+        path: str,
+        renderer,
+    ) -> list[FileChunk]:
+        """Try the whole subtree; on overflow split (leaf) or descend.
+        ``before``/``after`` are TOC fragments that bracket each emitted
+        chunk's content (chunk text = ``before + content + after``).
+        As we descend, the prefix grows with section headings already
+        passed and the suffix shrinks correspondingly.
+        """
+        if not node.text:
+            return []
+        if node.kind == "section":
+            heading_line = f"{'#' * node.level} {node.heading or ''}"
+            before_self = _toc_join(before, heading_line)
+        else:
+            before_self = before
+        if len(node.text) <= self.chunk_chars:
+            return [
+                self._make_chunk(
+                    before_self,
+                    node.text,
+                    after,
+                    node.start_line,
+                    node.end_line,
+                    path,
+                ),
+            ]
+        if node.kind == "body":
+            return self._split_leaf(node, before, after, path, renderer)
+        after_inside = _toc_join(node.desc_toc, after)
+        sub_tocs = [_subtree_toc(c) for c in node.children if c.kind == "section"]
+        chunks: list[FileChunk] = []
+        accumulated = before_self
+        sec_idx = 0
+        run: list[MdNode] = []
+        for c in node.children:
+            if c.kind == "section":
+                if run:
+                    chunks.extend(
+                        self._chunk_body_run(
+                            run,
+                            before_self,
+                            after_inside,
+                            path,
+                            renderer,
+                        ),
+                    )
+                    run = []
+                remaining = "\n\n".join(sub_tocs[sec_idx + 1 :])
+                chunks.extend(
+                    self._chunk_node(
+                        c,
+                        accumulated,
+                        _toc_join(remaining, after),
+                        path,
+                        renderer,
+                    ),
+                )
+                accumulated = _toc_join(accumulated, sub_tocs[sec_idx])
+                sec_idx += 1
+            else:
+                run.append(c)
+        if run:
+            chunks.extend(
+                self._chunk_body_run(
+                    run,
+                    before_self,
+                    after_inside,
+                    path,
+                    renderer,
+                ),
+            )
+        return chunks
+
+    def _chunk_body_run(
+        self,
+        run: list[MdNode],
+        before: str,
+        after: str,
+        path: str,
+        renderer,
+    ) -> list[FileChunk]:
+        """Greedy-pack consecutive body siblings under the same TOC slot.
+        No ``[Part X/N]`` markers — distinct blocks, not a leaf split.
+        Oversized single body recurses to ``_split_leaf``."""
+        composite_size = sum(len(b.text) for b in run) + 2 * max(0, len(run) - 1)
+        if composite_size <= self.chunk_chars:
+            return [
+                self._make_chunk(
+                    before,
+                    "\n\n".join(b.text for b in run),
+                    after,
+                    run[0].start_line,
+                    run[-1].end_line,
+                    path,
+                ),
+            ]
+
+        chunks: list[FileChunk] = []
+        bucket: list[MdNode] = []
+        bucket_chars = 0
+
+        def flush() -> None:
+            nonlocal bucket, bucket_chars
+            if not bucket:
+                return
+            chunks.append(
+                self._make_chunk(
+                    before,
+                    "\n\n".join(b.text for b in bucket),
+                    after,
+                    bucket[0].start_line,
+                    bucket[-1].end_line,
+                    path,
+                ),
+            )
+            bucket = []
+            bucket_chars = 0
+
+        for body in run:
+            if len(body.text) > self.chunk_chars:
+                flush()
+                chunks.extend(self._split_leaf(body, before, after, path, renderer))
+                continue
+            sep = 2 if bucket else 0
+            if bucket_chars + sep + len(body.text) > self.chunk_chars:
+                flush()
+                sep = 0
+            bucket.append(body)
+            bucket_chars += sep + len(body.text)
+        flush()
+        return chunks
+
+    # -- Leaf splitters: build (text, start, end) units, hand off to packer
+
+    def _split_leaf(
+        self,
+        body: MdNode,
+        before: str,
+        after: str,
+        path: str,
+        renderer,
+    ) -> list[FileChunk]:
+        from mistletoe.block_token import (
+            CodeFence,
+            List,
+            Table,
+        )
+
+        block = body.block
+        if isinstance(block, Table):
+            return self._split_table(body, before, after, path)
+        if isinstance(block, CodeFence):
+            return self._split_code(body, before, after, path)
+        if isinstance(block, List):
+            return self._split_list(body, before, after, path, renderer)
+        return self._split_lines(body, before, after, path)
+
+    def _split_table(
+        self,
+        body: MdNode,
+        before: str,
+        after: str,
+        path: str,
+    ) -> list[FileChunk]:
+        """Repeat header + separator on every chunk."""
+        from mistletoe.block_token import TableRow
+
+        lines = body.text.split("\n")
+        header, data = "\n".join(lines[:2]), lines[2:]
+        rows = [r for r in (body.block.children or []) if isinstance(r, TableRow)]
+        base = body.start_line + 2
+
+        def line_of(i: int) -> int:
+            return rows[i].line_number if i < len(rows) and rows[i].line_number else base + i
+
+        units = [(text, line_of(i), line_of(i)) for i, text in enumerate(data)]
+        return self._emit_packed(
+            units,
+            before,
+            after,
+            path,
+            joiner="\n",
+            wrap=f"{header}\n{{inner}}",
+        )
+
+    def _split_code(
+        self,
+        body: MdNode,
+        before: str,
+        after: str,
+        path: str,
+    ) -> list[FileChunk]:
+        """Repeat fence opener + closer on every chunk."""
+        code = body.block
+        indent = " " * (code.indentation or 0)
+        fence = f"{indent}{code.delimiter}"
+        opener = f"{fence}{code.info_string or ''}"
+        raw = (code.children[0].content if code.children else "").rstrip("\n")
+        if not raw:
+            return []
+        start = body.start_line + 1
+        units = [(indent + ln, start + i, start + i) for i, ln in enumerate(raw.split("\n"))]
+        return self._emit_packed(
+            units,
+            before,
+            after,
+            path,
+            joiner="\n",
+            wrap=f"{opener}\n{{inner}}\n{fence}",
+            allow_empty=True,
+        )
+
+    def _split_list(
+        self,
+        body: MdNode,
+        before: str,
+        after: str,
+        path: str,
+        renderer,
+    ) -> list[FileChunk]:
+        """Pack list items; oversized items emit alone (overflow accepted)."""
+        from mistletoe.block_token import ListItem
+
+        items = [c for c in (body.block.children or []) if isinstance(c, ListItem)]
+        if not items:
+            return self._split_lines(body, before, after, path)
+        units: list[tuple[str, int, int]] = []
+        for it in items:
+            text = renderer.render(it).rstrip("\n")
+            if not text:
+                continue
+            line = it.line_number or body.start_line
+            units.append((text, line, line + text.count("\n")))
+        return self._emit_packed(
+            units,
+            before,
+            after,
+            path,
+            joiner="\n",
+            wrap="{inner}",
+        )
+
+    def _split_lines(
+        self,
+        body: MdNode,
+        before: str,
+        after: str,
+        path: str,
+    ) -> list[FileChunk]:
+        """Last-resort line-greedy split for paragraphs / quotes / html."""
+        start = body.start_line
+        units = [(line, start + i, start + i) for i, line in enumerate(body.text.split("\n"))]
+        return self._emit_packed(
+            units,
+            before,
+            after,
+            path,
+            joiner="\n",
+            wrap="{inner}",
+        )
+
+    def _emit_packed(
+        self,
+        units: list[tuple[str, int, int]],
+        before: str,
+        after: str,
+        path: str,
+        joiner: str,
+        wrap: str,
+        allow_empty: bool = False,
+    ) -> list[FileChunk]:
+        """Greedy-pack units into ``wrap`` envelopes; emit each piece.
+
+        Envelope (table header, code fence) counts against ``chunk_chars``;
+        TOC (when on) is additive prefix/suffix downstream. Oversized
+        units overflow rather than truncate. Multi-piece outputs get
+        ``[Part X/N]`` markers; single pieces don't.
+        """
+        envelope = len(wrap.replace("{inner}", ""))
+        budget = max(64, self.chunk_chars - envelope)
+        sep_len = len(joiner)
+
+        parts: list[tuple[str, int, int]] = []
+        bucket: list[tuple[str, int, int]] = []
+        bucket_chars = 0
+
+        def flush() -> None:
+            nonlocal bucket, bucket_chars
+            if not bucket:
+                return
+            inner = joiner.join(t for t, _, _ in bucket)
+            parts.append((inner, bucket[0][1], bucket[-1][2]))
+            bucket = []
+            bucket_chars = 0
+
+        for text, s, e in units:
+            if not text and not allow_empty:
+                continue
+            sep = sep_len if bucket else 0
+            if bucket_chars + sep + len(text) > budget:
+                flush()
+                sep = 0
+            bucket.append((text, s, e))
+            bucket_chars += sep + len(text)
+        flush()
+
+        total = len(parts)
+        return [
+            self._make_chunk(
+                before,
+                (
+                    f"[Part {idx}/{total}]\n\n{wrap.replace('{inner}', inner)}"
+                    if total > 1
+                    else wrap.replace("{inner}", inner)
+                ),
+                after,
+                s,
+                e,
+                path,
+            )
+            for idx, (inner, s, e) in enumerate(parts, 1)
+        ]
+
+    # -- Emit -------------------------------------------------------------
+
+    def _make_chunk(
+        self,
+        before: str,
+        content: str,
+        after: str,
+        start_line: int,
+        end_line: int,
+        path: str,
+    ) -> FileChunk:
+        """Build one ``FileChunk`` — text is ``before + content + after``
+        when ``embed_toc``, otherwise just ``content``."""
+        text = _toc_join(before, content, after) if self.embed_toc else content
+        return FileChunk(
+            path=path,
+            start_line=start_line,
+            end_line=end_line,
+            text=text,
+        ).set_hash_id()

reme4/components/file_store/__init__.py

@@ -0,0 +1,14 @@
+"""File store module.
+
+In-memory + JSONL backend for the (file → chunks) graph. Subclass
+`BaseFileStore` to add other backends; only `LocalFileStore` is
+shipped today.
+"""
+
+from .base_file_store import BaseFileStore
+from .local_file_store import LocalFileStore
+
+__all__ = [
+    "BaseFileStore",
+    "LocalFileStore",
+]

reme4/components/file_store/base_file_store.py

@@ -0,0 +1,100 @@
+"""Abstract base for file store backends."""
+
+from abc import abstractmethod
+
+from ..base_component import BaseComponent
+from ..embedding import BaseEmbeddingModel
+from ..file_graph import BaseFileGraph
+from ..keyword_index import BaseKeywordIndex
+from ...enumeration import ComponentEnum
+from ...schema import FileChunk, FileNode, FileLink
+
+
+class BaseFileStore(BaseComponent):
+    """Abstract base for file store backends."""
+
+    component_type = ComponentEnum.FILE_STORE
+
+    def __init__(
+        self,
+        store_name: str,
+        embedding_model: str = "default",
+        keyword_index: str = "default",
+        file_graph: str = "default",
+        store_version: str = "v1",
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        from ..embedding import OpenAIEmbeddingModel
+        from ..file_graph import LocalFileGraph
+        from ..keyword_index import BM25Index
+
+        self.store_name = store_name or self.name
+        self.store_version = store_version
+        if not embedding_model and not keyword_index:
+            raise ValueError("At least one of embedding_model or keyword_index must be set.")
+
+        self.embedding_model = self.bind(embedding_model, BaseEmbeddingModel, default_factory=OpenAIEmbeddingModel)
+        self.keyword_index = self.bind(keyword_index, BaseKeywordIndex, default_factory=BM25Index)
+        self.file_graph = self.bind(file_graph, BaseFileGraph, default_factory=LocalFileGraph)
+        self.store_path = self.working_metadata_path / self.component_type.value / store_name
+        self.store_path.mkdir(parents=True, exist_ok=True)
+
+    async def _start(self) -> None:
+        """Probe embedding model; disable vector capability if it fails."""
+        if self.embedding_model is None:
+            return
+        if not await self.embedding_model.health_check():
+            self.logger.warning(f"{self.store_name}: embedding unhealthy, vector disabled")
+            self.embedding_model = None
+
+    def _disable_embedding(self, reason: str) -> None:
+        """Drop embedding after a runtime failure; keyword search still works."""
+        if self.embedding_model is None:
+            return
+        self.logger.error(f"{self.store_name}: embedding disabled, {reason}")
+        self.embedding_model = None
+
+    async def upsert_file(
+        self,
+        file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]],
+    ) -> None:
+        """Upsert a file and its chunks into the store."""
+
+    async def delete_by_path(self, path: str | list[str]) -> None:
+        """Delete files by their paths from the store."""
+
+    async def clear(self):
+        """Clear the store of all files and chunks."""
+
+    @abstractmethod
+    async def vector_search(self, query: str, limit: int, search_filter: dict) -> list[FileChunk]:
+        """Perform vector similarity search."""
+
+    @abstractmethod
+    async def keyword_search(self, query: str, limit: int, search_filter: dict) -> list[FileChunk]:
+        """Perform full-text keyword search."""
+
+    async def rebuild_links(self) -> None:
+        """Rebuild all edges from each node's link payload."""
+        if not self.file_graph:
+            raise RuntimeError("file_graph is required for delete_by_path")
+        return await self.file_graph.rebuild_links()
+
+    async def get_nodes(self, paths: list[str]) -> list[FileNode]:
+        """Return file nodes for the given paths (missing paths are skipped)."""
+        if not self.file_graph:
+            raise RuntimeError("file_graph is required for get_nodes")
+        return await self.file_graph.get_nodes(paths)
+
+    async def get_outlinks(self, path: str) -> list[FileLink]:
+        """Return outgoing links for *path*."""
+        if not self.file_graph:
+            raise RuntimeError("file_graph is required for delete_by_path")
+        return await self.file_graph.get_outlinks(path)
+
+    async def get_inlinks(self, path: str) -> list[FileLink]:
+        """Return incoming links for *path*."""
+        if not self.file_graph:
+            raise RuntimeError("file_graph is required for delete_by_path")
+        return await self.file_graph.get_inlinks(path)

reme4/components/file_store/local_file_store.py

@@ -0,0 +1,177 @@
+"""In-memory file store with JSONL persistence on close."""
+
+import aiofiles
+import numpy as np
+
+from .base_file_store import BaseFileStore
+from ..component_registry import R
+from ...schema import FileChunk, FileNode
+from ...utils import batch_cosine_similarity
+
+
+@R.register("local")
+class LocalFileStore(BaseFileStore):
+    """In-memory file store with deferred JSONL persistence."""
+
+    def __init__(self, encoding: str = "utf-8", **kwargs):
+        super().__init__(**kwargs)
+        self.encoding = encoding
+        self.file_chunks: dict[str, FileChunk] = {}
+        self.chunks_path = self.store_path / f"file_chunks_{self.store_version}.jsonl"
+
+    # Lifecycle
+
+    async def _start(self) -> None:
+        await super()._start()
+        await self.load()
+
+    async def _close(self) -> None:
+        await self.dump()
+        self.file_chunks.clear()
+        await super()._close()
+
+    async def load(self) -> None:
+        """Load chunks from JSONL file into memory."""
+        if not self.chunks_path.exists():
+            return
+        try:
+            async with aiofiles.open(self.chunks_path, encoding=self.encoding) as f:
+                async for line in f:
+                    line = line.strip()
+                    if line:
+                        chunk = FileChunk.model_validate_json(line)
+                        self.file_chunks[chunk.id] = chunk
+            self.logger.info(f"Loaded {len(self.file_chunks)} chunks from {self.chunks_path}")
+        except Exception as e:
+            self.logger.exception(f"Failed to load {self.chunks_path}: {e}")
+
+    async def dump(self) -> None:
+        """Persist chunks to JSONL via atomic rename, then cascade to keyword_index and file_graph."""
+        try:
+            tmp = self.chunks_path.with_suffix(".tmp")
+            async with aiofiles.open(tmp, "w", encoding=self.encoding) as f:
+                await f.write("\n".join(c.model_dump_json() for c in self.file_chunks.values()))
+            tmp.replace(self.chunks_path)
+            self.logger.info(f"Saved {len(self.file_chunks)} chunks to {self.chunks_path}")
+        except Exception as e:
+            self.logger.exception(f"Failed to write {self.chunks_path}: {e}")
+        if self.keyword_index:
+            await self.keyword_index.dump()
+        if self.file_graph:
+            await self.file_graph.dump()
+
+    # Base class interface
+
+    async def upsert_file(
+        self,
+        file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]],
+    ) -> None:
+        if not self.file_graph:
+            raise RuntimeError("file_graph is required for upsert_file")
+        if isinstance(file, tuple):
+            file = [file]
+
+        old_map = {n.path: n for n in await self.file_graph.get_nodes([node.path for node, _ in file])}
+
+        new_nodes: list[FileNode] = []
+        needs_embed: list[FileChunk] = []
+        keyword_docs: dict[str, str] = {}
+        for node, chunks in file:
+            old_node: FileNode | None = old_map.get(node.path)
+            cached = {}
+            if old_node and self.embedding_model:
+                for cid in old_node.chunk_ids:
+                    old = self.file_chunks.pop(cid, None)
+                    if old and old.embedding is not None:
+                        cached[cid] = old.embedding
+
+            node.chunk_ids = []
+            for c in chunks:
+                if self.embedding_model and c.embedding is None:
+                    if c.id in cached:
+                        c.embedding = cached[c.id]
+                    elif c.text:
+                        needs_embed.append(c)
+                node.chunk_ids.append(c.id)
+                self.file_chunks[c.id] = c
+                if c.text:
+                    keyword_docs[c.id] = c.text
+            new_nodes.append(node)
+
+        await self.file_graph.upsert_nodes(new_nodes)
+        if needs_embed and self.embedding_model:
+            try:
+                await self.embedding_model.get_node_embeddings(needs_embed)
+            except Exception as e:
+                self._disable_embedding(f"upsert: {type(e).__name__}: {e}")
+        if self.keyword_index and keyword_docs:
+            await self.keyword_index.add_docs(keyword_docs)
+
+    async def delete_by_path(self, path: str | list[str]) -> None:
+        if not self.file_graph:
+            raise RuntimeError("file_graph is required for delete_by_path")
+        if isinstance(path, str):
+            path = [path]
+        nodes = await self.file_graph.get_nodes(path)
+        if not nodes:
+            return
+        deleted_chunk_ids = [cid for n in nodes for cid in n.chunk_ids]
+        for cid in deleted_chunk_ids:
+            self.file_chunks.pop(cid, None)
+        await self.file_graph.delete_nodes([n.path for n in nodes])
+        if self.keyword_index and deleted_chunk_ids:
+            await self.keyword_index.delete_docs(deleted_chunk_ids)
+
+    async def clear(self) -> None:
+        if not self.file_graph:
+            raise RuntimeError("file_graph is required for clear")
+        self.file_chunks.clear()
+        self.chunks_path.unlink(missing_ok=True)
+        if self.keyword_index:
+            await self.keyword_index.clear()
+        await self.file_graph.clear()
+
+    # Search
+
+    async def vector_search(self, query: str, limit: int, search_filter: dict) -> list[FileChunk]:
+        if self.embedding_model is None or not query:
+            return []
+
+        try:
+            query_embedding = await self.embedding_model.get_embedding(query)
+        except Exception as e:
+            self._disable_embedding(f"search: {type(e).__name__}: {e}")
+            return []
+        if query_embedding is None:
+            return []
+
+        candidates = [c for c in self.file_chunks.values() if c.embedding is not None]
+        if not candidates:
+            return []
+
+        candidate_embeddings = np.stack([c.embedding for c in candidates])
+        similarities = batch_cosine_similarity(query_embedding.reshape(1, -1), candidate_embeddings)[0]
+
+        results = [
+            c.model_copy(update={"scores": {"vector": float(s), "score": float(s)}})
+            for c, s in zip(candidates, similarities)
+        ]
+        results.sort(key=lambda r: r.score, reverse=True)
+        return results[:limit]
+
+    async def keyword_search(self, query: str, limit: int, search_filter: dict) -> list[FileChunk]:
+        if not self.keyword_index:
+            return []
+
+        query = query.strip()
+        if not query:
+            return []
+
+        doc_id_score_dict = await self.keyword_index.retrieve(query, limit=limit)
+        results = []
+        for doc_id, score in doc_id_score_dict.items():
+            chunk = self.file_chunks.get(doc_id)
+            if chunk:
+                results.append(chunk.model_copy(update={"scores": {"keyword": score, "score": score}}))
+
+        return results

reme4/components/file_watcher/__init__.py

@@ -0,0 +1,9 @@
+"""File watcher implementations for monitoring file system changes."""
+
+from .base_file_watcher import BaseFileWatcher
+from .lite_file_watcher import LiteFileWatcher
+
+__all__ = [
+    "BaseFileWatcher",
+    "LiteFileWatcher",
+]

reme4/components/file_watcher/base_file_watcher.py

@@ -0,0 +1,118 @@
+"""Abstract base for file watchers."""
+
+import asyncio
+from abc import abstractmethod
+from pathlib import Path
+
+from watchfiles import Change
+
+from ..base_component import BaseComponent
+from ..file_parser import BaseFileParser
+from ..file_store import BaseFileStore
+from ...enumeration import ComponentEnum
+
+
+class BaseFileWatcher(BaseComponent):
+    """Abstract base for file watchers. Subclasses implement watch_loop and event handlers."""
+
+    component_type = ComponentEnum.FILE_WATCHER
+
+    def __init__(
+        self,
+        watch_paths: list[str] | str,
+        suffix_filters: list[str] | None = None,
+        recursive: bool = True,
+        force_polling: bool = True,
+        debounce: int = 2000,
+        poll_delay_ms: int = 2000,
+        file_store: str = "default",
+        file_parser: str = "default",
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        from ..file_parser import DefaultFileParser
+        from ..file_store import LocalFileStore
+
+        watch_paths = [watch_paths] if isinstance(watch_paths, str) else watch_paths
+        base = self.working_path
+        self.watch_paths: list[Path] = [base / x for x in watch_paths if (base / x).exists()]
+        self.suffix_filters: list[str] = suffix_filters or ["md"]
+        self.recursive: bool = recursive
+        self.force_polling: bool = force_polling
+        self.debounce: int = debounce
+        self.poll_delay_ms: int = poll_delay_ms
+        self.file_store = self.bind(file_store, BaseFileStore, default_factory=LocalFileStore)
+        self.file_parser = self.bind(file_parser, BaseFileParser, default_factory=DefaultFileParser)
+        self._stop_event: asyncio.Event = asyncio.Event()
+        self._background_task: asyncio.Task | None = None
+        self._retry_interval: float = 10
+
+    async def _start(self):
+        self._stop_event = asyncio.Event()
+        self._background_task = asyncio.create_task(self._background_run())
+        self.logger.info(f"Started watching: {[str(p) for p in self.watch_paths]}")
+
+    async def _background_run(self):
+        """Sync store then enter watch loop."""
+        await self.update_store()
+        await self.watch_loop()
+
+    async def _close(self):
+        self._stop_event.set()
+        if self._background_task:
+            await self._background_task
+        self.logger.info("Stopped watching")
+
+    def watch_filter(self, _change: Change, path: str) -> bool:
+        """Return True if the file suffix matches the filter list."""
+        if not self.suffix_filters:
+            return True
+        return any(path.endswith("." + s.strip(".")) for s in self.suffix_filters)
+
+    def _get_relative_path(self, path: str | Path) -> str:
+        """Return path relative to working_dir, or absolute path if outside."""
+        file_path = Path(path).absolute()
+        try:
+            return str(file_path.relative_to(self.working_path.absolute()))
+        except ValueError:
+            return str(file_path)
+
+    def _get_absolute_path(self, path: str | Path) -> Path:
+        """Return absolute path; relative paths are resolved against working_dir."""
+        p = Path(path)
+        return p if p.is_absolute() else self.working_path / p
+
+    async def scan_existing_files(self) -> dict[str, Path]:
+        """Collect watchable files under watch_paths as {relative_path: absolute_path}."""
+        files: dict[str, Path] = {}
+        for path in self.watch_paths:
+            if not path.exists():
+                continue
+            candidates = [path] if path.is_file() else (path.rglob("*") if self.recursive else path.iterdir())
+            for p in candidates:
+                if p.is_file() and self.watch_filter(Change.added, str(p)):
+                    files[self._get_relative_path(p)] = p.absolute()
+        return files
+
+    @abstractmethod
+    async def watch_loop(self):
+        """Watch for file changes and dispatch events."""
+
+    @abstractmethod
+    async def update_store(self, dump: bool = True) -> dict[str, int]:
+        """Sync the store with watch_paths; dump store if any changes and dump=True.
+
+        Returns counts {"added": int, "modified": int, "deleted": int}.
+        """
+
+    @abstractmethod
+    async def on_added(self, path: str | list[str]):
+        """Handle file added event (relative paths)."""
+
+    @abstractmethod
+    async def on_modified(self, path: str | list[str]):
+        """Handle file modified event (relative paths)."""
+
+    @abstractmethod
+    async def on_deleted(self, path: str | list[str]):
+        """Handle file deleted event (relative paths)."""

reme4/components/file_watcher/lite_file_watcher.py

@@ -0,0 +1,129 @@
+"""Polling-based file watcher using watchfiles."""
+
+import asyncio
+
+from watchfiles import Change, awatch
+
+from .base_file_watcher import BaseFileWatcher
+from ..component_registry import R
+from ...schema import FileChunk, FileNode
+
+
+@R.register("lite")
+class LiteFileWatcher(BaseFileWatcher):
+    """Polling-based file watcher using watchfiles awatch."""
+
+    async def _interruptible_sleep(self):
+        """Sleep until stop or timeout, whichever comes first."""
+        try:
+            await asyncio.wait_for(self._stop_event.wait(), timeout=self._retry_interval)
+        except asyncio.TimeoutError:
+            pass
+
+    async def watch_loop(self):
+        if not self.watch_paths:
+            self.logger.warning("No watch paths specified")
+            return
+
+        while not self._stop_event.is_set():
+            valid_paths = [p for p in self.watch_paths if p.exists()]
+            if not valid_paths:
+                self.logger.warning(f"No valid paths, retrying in {self._retry_interval}s...")
+                await self._interruptible_sleep()
+                continue
+
+            invalid = set(self.watch_paths) - set(valid_paths)
+            if invalid:
+                self.logger.warning(f"Skipping invalid paths: {[str(p) for p in invalid]}")
+
+            try:
+                self.logger.info(f"Watching: {[str(p) for p in valid_paths]}")
+                async for changes in awatch(
+                    *valid_paths,
+                    watch_filter=self.watch_filter,
+                    recursive=self.recursive,
+                    force_polling=self.force_polling,
+                    debounce=self.debounce,
+                    poll_delay_ms=self.poll_delay_ms,
+                    stop_event=self._stop_event,
+                ):
+                    if self._stop_event.is_set():
+                        break
+                    await self._dispatch_changes(changes)
+            except Exception:
+                self.logger.exception(f"Watch error, retrying in {self._retry_interval}s...")
+                if not self._stop_event.is_set():
+                    await self._interruptible_sleep()
+
+    async def _dispatch_changes(self, changes: set[tuple[Change, str]]):
+        """Classify raw changes and dispatch to event handlers."""
+        buckets: dict[Change, list[str]] = {Change.added: [], Change.modified: [], Change.deleted: []}
+        for c, p in changes:
+            if c in buckets:
+                buckets[c].append(self._get_relative_path(p))
+        for change, handler, label in (
+            (Change.added, self.on_added, "added"),
+            (Change.modified, self.on_modified, "modified"),
+            (Change.deleted, self.on_deleted, "deleted"),
+        ):
+            if buckets[change]:
+                self.logger.info(f"Detected {len(buckets[change])} {label} file(s)")
+                await handler(buckets[change])
+
+    async def update_store(self, dump: bool = True) -> dict[str, int]:
+        if self.file_store is None:
+            raise ValueError("file_store is not initialized!")
+
+        existing: dict[str, float] = {
+            rel: abs_p.stat().st_mtime for rel, abs_p in (await self.scan_existing_files()).items()
+        }
+        indexed: dict[str, float] = {n.path: n.st_mtime for n in await self.file_store.file_graph.get_nodes()}
+
+        to_delete = list(indexed.keys() - existing.keys())
+        to_add = list(existing.keys() - indexed.keys())
+        to_modify = [p for p in existing.keys() & indexed.keys() if existing[p] != indexed[p]]
+
+        if to_modify:
+            self.logger.info(f"Updating {len(to_modify)} modified file(s)")
+            await self.on_modified(to_modify)
+        if to_delete:
+            self.logger.info(f"Removing {len(to_delete)} deleted file(s)")
+            await self.on_deleted(to_delete)
+        if to_add:
+            self.logger.info(f"Indexing {len(to_add)} new file(s)")
+            await self.on_added(to_add)
+
+        changed = bool(to_add or to_modify or to_delete)
+        if not changed:
+            self.logger.info("Store is up to date")
+        if dump and changed:
+            await self.file_store.dump()
+        return {"added": len(to_add), "modified": len(to_modify), "deleted": len(to_delete)}
+
+    async def _parse_and_upsert(self, paths: list[str], action: str):
+        """Parse files and upsert into store. Shared by on_added / on_modified."""
+        if self.file_parser is None or self.file_store is None:
+            raise RuntimeError("file_parser or file_store is not initialized!")
+
+        parsed: list[tuple[FileNode, list[FileChunk]]] = []
+        for rel in paths:
+            abs_path = self._get_absolute_path(rel)
+            if abs_path.is_file():
+                self.logger.info(f"{action} file: {rel}")
+                parsed.append(await self.file_parser.parse(abs_path))
+        if parsed:
+            await self.file_store.delete_by_path([n.path for n, _ in parsed])
+            await self.file_store.upsert_file(parsed)
+
+    async def on_added(self, path: str | list[str]):
+        await self._parse_and_upsert([path] if isinstance(path, str) else path, "Adding")
+
+    async def on_modified(self, path: str | list[str]):
+        await self._parse_and_upsert([path] if isinstance(path, str) else path, "Updating")
+
+    async def on_deleted(self, path: str | list[str]):
+        if self.file_store is None:
+            raise RuntimeError("file_store is not initialized!")
+        paths = [path] if isinstance(path, str) else path
+        self.logger.info(f"Deleting {len(paths)} file(s)")
+        await self.file_store.delete_by_path(paths)

reme4/components/job/__init__.py

@@ -0,0 +1,6 @@
+"""Job components for executing workflows."""
+
+from .base_job import BaseJob
+from .stream_job import StreamJob
+
+__all__ = ["BaseJob", "StreamJob"]

reme4/components/job/base_job.py

@@ -0,0 +1,54 @@
+"""Base job component for sequential step execution."""
+
+from ..base_component import BaseComponent
+from ..component_registry import R
+from ..runtime_context import RuntimeContext
+from ...enumeration import ComponentEnum
+from ...schema import ComponentConfig, Response
+
+
+@R.register("base")
+class BaseJob(BaseComponent):
+    """Job that executes steps sequentially and returns a Response."""
+
+    component_type = ComponentEnum.JOB
+
+    def __init__(self, description: str, parameters: dict, steps: list[ComponentConfig | dict], **kwargs):
+        super().__init__(**kwargs)
+        self.description = description
+        self.parameters = parameters or {}
+        self.step_configs = steps or []
+
+        from ...steps import BaseStep
+
+        self.step_components: list[BaseStep] = []
+
+    async def _start(self) -> None:
+        """Resolve step configs into instantiated step components."""
+        assert self.app_context is not None, "app_context must be provided"
+        for raw in self.step_configs:
+            config = raw if isinstance(raw, ComponentConfig) else ComponentConfig(**raw)
+            if not config.backend:
+                raise ValueError("Step is missing the required 'backend' field")
+            step_cls = R.get(ComponentEnum.STEP, config.backend)
+            if not step_cls:
+                raise ValueError(f"Unregistered backend '{config.backend}' of type '{ComponentEnum.STEP}'")
+            params = config.model_dump()
+            params["app_context"] = self.app_context
+            self.step_components.append(step_cls(**params))
+
+    async def _close(self) -> None:
+        """Release all step components."""
+        self.step_components.clear()
+
+    async def __call__(self, **kwargs) -> Response:
+        """Execute all steps in order and return the final response."""
+        context = RuntimeContext(**kwargs)
+        try:
+            for step in self.step_components:
+                await step(context)
+        except Exception as e:
+            self.logger.exception(f"Failed to execute job: {e}")
+            context.response.success = False
+            context.response.answer = str(e)
+        return context.response

reme4/components/job/stream_job.py

@@ -0,0 +1,21 @@
+"""Streaming job for real-time output delivery."""
+
+from .base_job import BaseJob
+from ..component_registry import R
+from ..runtime_context import RuntimeContext
+from ...enumeration import ChunkEnum
+
+
+@R.register("stream")
+class StreamJob(BaseJob):
+    """Job that streams chunks to a queue instead of returning a Response."""
+
+    async def __call__(self, **kwargs) -> None:
+        """Execute steps and stream output; errors are sent as ERROR chunks."""
+        context = RuntimeContext(**kwargs)
+        try:
+            for step in self.step_components:
+                await step(context)
+        except Exception as e:
+            await context.add_stream_string(str(e), ChunkEnum.ERROR)
+        await context.add_stream_done()

reme4/components/keyword_index/__init__.py

@@ -0,0 +1,6 @@
+"""Keyword index components."""
+
+from .base_keyword_index import BaseKeywordIndex
+from .bm25_index import BM25Index
+
+__all__ = ["BaseKeywordIndex", "BM25Index"]

reme4/components/keyword_index/base_keyword_index.py

@@ -0,0 +1,70 @@
+"""Abstract base class for keyword index implementations."""
+
+from abc import abstractmethod
+from pathlib import Path
+
+from ..base_component import BaseComponent
+from ..tokenizer import BaseTokenizer
+from ...enumeration import ComponentEnum
+
+
+class BaseKeywordIndex(BaseComponent):
+    """Abstract base class for keyword index implementations."""
+
+    component_type = ComponentEnum.KEYWORD_INDEX
+
+    def __init__(self, tokenizer: str = "default", index_version: str = "v1", **kwargs):
+        super().__init__(**kwargs)
+        from ..tokenizer import RegexTokenizer
+
+        self.tokenizer = self.bind(tokenizer, BaseTokenizer, default_factory=RegexTokenizer)
+        self.index_version = index_version
+        self.index_path = self.working_metadata_path / self.component_type.value
+        self.index_path.mkdir(parents=True, exist_ok=True)
+
+    async def _start(self) -> None:
+        """Load existing index from disk if available."""
+        await self.load()
+
+    async def _close(self) -> None:
+        """Save index to disk on shutdown."""
+        await self.dump()
+
+    @property
+    def index_file(self) -> Path:
+        """Return the pickle file path derived from tokenizer name."""
+        if self.tokenizer is None:
+            raise RuntimeError("Tokenizer not initialized. Call start() first.")
+        name = type(self.tokenizer).__name__.replace("Tokenizer", "").lower()
+        return self.index_path / f"bm25_{name}_{self.index_version}.pkl"
+
+    def _tokenize(self, text: str) -> list[str]:
+        """Tokenize a text string into tokens."""
+        if self.tokenizer is None:
+            raise RuntimeError("Tokenizer not initialized. Call start() first.")
+        return self.tokenizer.tokenize([text])[0]
+
+    @abstractmethod
+    async def add_docs(self, docs_dict: dict[str, str]) -> None:
+        """Index or update documents. Mapping of doc_id to content."""
+
+    @abstractmethod
+    async def delete_docs(self, doc_ids: list[str]) -> None:
+        """Remove documents by their IDs."""
+
+    @abstractmethod
+    async def retrieve(self, query: str, limit: int = 3) -> dict[str, float]:
+        """Search documents. Returns {doc_id: score} sorted descending."""
+
+    @abstractmethod
+    async def clear(self) -> None:
+        """Reset index to empty state."""
+
+    async def reset_index(self, docs_dict: dict[str, str]) -> None:
+        """Clear index, re-add all documents, and persist."""
+        await self.clear()
+        await self.add_docs(docs_dict)
+        await self.dump()
+
+    async def optimize_index(self) -> None:
+        """Optimize index for performance. Override in subclass if needed."""

reme4/components/keyword_index/bm25_index.py

@@ -0,0 +1,206 @@
+"""BM25 search engine with persistent index support.
+
+Implements Okapi BM25 ranking with an inverted index for efficient
+document lookup, incremental updates, and pickle-based persistence.
+"""
+
+import math
+import pickle
+from collections import Counter
+from typing import TypedDict
+
+from .base_keyword_index import BaseKeywordIndex
+from ..component_registry import R
+
+
+class DocMeta(TypedDict):
+    """Per-document metadata: token count and unique token ID set."""
+
+    len: int
+    token_ids: set[int]
+
+
+@R.register("bm25")
+class BM25Index(BaseKeywordIndex):
+    """BM25 search engine with file-based persistence.
+
+    Args:
+        k1: Term frequency saturation parameter (default 1.5).
+        b: Document length normalization parameter (default 0.75).
+    """
+
+    def __init__(self, k1: float = 1.5, b: float = 0.75, **kwargs):
+        super().__init__(**kwargs)
+        self.k1 = k1
+        self.b = b
+        self.vocab: dict[str, int] = {}  # token -> token_id
+        self.inverted_index: dict[int, dict[str, int]] = {}  # token_id -> {doc_id: tf}
+        self.doc_meta: dict[str, DocMeta] = {}  # doc_id -> metadata
+        self.total_len: int = 0
+        self._idf_cache: dict[int, float] = {}
+
+    # -- Properties -----------------------------------------------------------
+
+    @property
+    def n_docs(self) -> int:
+        """Number of indexed documents."""
+        return len(self.doc_meta)
+
+    @property
+    def avg_len(self) -> float:
+        """Average document length in tokens."""
+        return self.total_len / self.n_docs if self.n_docs > 0 else 0.0
+
+    # -- Internal helpers -----------------------------------------------------
+
+    def _tokens_to_ids(self, tokens: list[str]) -> list[int]:
+        """Map tokens to integer IDs, assigning new IDs on first encounter."""
+        ids = []
+        for token in tokens:
+            token = token.strip()
+            if token:
+                ids.append(self.vocab.setdefault(token, len(self.vocab)))
+        return ids
+
+    def _remove_doc(self, doc_id: str) -> None:
+        """Remove a single document from all internal structures."""
+        if doc_id not in self.doc_meta:
+            return
+        meta = self.doc_meta[doc_id]
+        self.total_len -= meta["len"]
+        for tid in meta["token_ids"]:
+            if tid in self.inverted_index:
+                self.inverted_index[tid].pop(doc_id, None)
+                if not self.inverted_index[tid]:
+                    del self.inverted_index[tid]
+        del self.doc_meta[doc_id]
+
+    def _get_idf(self, token_id: int) -> float:
+        """Compute and cache IDF for a token ID."""
+        if token_id in self._idf_cache:
+            return self._idf_cache[token_id]
+        df = len(self.inverted_index.get(token_id, {}))
+        self._idf_cache[token_id] = math.log(1 + (self.n_docs - df + 0.5) / (df + 0.5)) if df else 0.0
+        return self._idf_cache[token_id]
+
+    # -- Public API -----------------------------------------------------------
+
+    async def add_docs(self, docs_dict: dict[str, str]) -> None:
+        """Index or update multiple documents. Mapping of doc_id to content."""
+        for doc_id, content in docs_dict.items():
+            if doc_id in self.doc_meta:
+                self._remove_doc(doc_id)
+            tokens = self._tokenize(content)
+            if not tokens:
+                continue
+            token_ids = self._tokens_to_ids(tokens)
+            token_counts = Counter(token_ids)
+            for tid, tf in token_counts.items():
+                self.inverted_index.setdefault(tid, {})[doc_id] = tf
+            self.doc_meta[doc_id] = {"len": len(token_ids), "token_ids": set(token_counts)}
+            self.total_len += len(token_ids)
+        self._idf_cache = {}
+
+    async def delete_docs(self, doc_ids: list[str]) -> None:
+        """Remove documents by their IDs."""
+        for doc_id in doc_ids:
+            self._remove_doc(doc_id)
+        self._idf_cache = {}
+
+    async def retrieve(self, query: str, limit: int = 3) -> dict[str, float]:
+        """Search documents. Returns {doc_id: score} sorted descending."""
+        query_ids = [self.vocab[t] for t in self._tokenize(query) if t in self.vocab]
+        if not query_ids or self.n_docs == 0:
+            return {}
+
+        scores: dict[str, float] = {}
+        avg_len = self.avg_len
+        for tid in query_ids:
+            if tid not in self.inverted_index:
+                continue
+            idf = self._get_idf(tid)
+            for doc_id, tf in self.inverted_index[tid].items():
+                doc_len = self.doc_meta[doc_id]["len"]
+                tf_score = tf * (self.k1 + 1) / (tf + self.k1 * (1 - self.b + self.b * doc_len / avg_len))
+                scores[doc_id] = scores.get(doc_id, 0.0) + idf * tf_score
+
+        return dict(sorted(scores.items(), key=lambda x: x[1], reverse=True)[:limit]) if scores else {}
+
+    async def dump(self) -> None:
+        """Persist index to disk via pickle (atomic rename)."""
+        try:
+            tmp = self.index_file.with_suffix(".tmp")
+            with open(tmp, "wb") as f:
+                pickle.dump(
+                    {
+                        "vocab": self.vocab,
+                        "inverted_index": self.inverted_index,
+                        "doc_meta": self.doc_meta,
+                        "total_len": self.total_len,
+                        "k1": self.k1,
+                        "b": self.b,
+                    },
+                    f,
+                )
+            tmp.replace(self.index_file)
+            self.logger.info(f"Saved {self.n_docs} docs to {self.index_file}")
+        except Exception as e:
+            self.logger.exception(f"Failed to write {self.index_file}: {e}")
+
+    async def load(self) -> None:
+        """Load index from disk. No-op if file missing; clears index on corruption."""
+        if not self.index_file.exists():
+            return
+        try:
+            with open(self.index_file, "rb") as f:
+                data = pickle.load(f)
+            self.vocab = data["vocab"]
+            self.inverted_index = data["inverted_index"]
+            self.doc_meta = data["doc_meta"]
+            self.total_len = data.get("total_len", 0)
+            self.k1 = data.get("k1", 1.5)
+            self.b = data.get("b", 0.75)
+            self._idf_cache = {}
+            self.logger.info(f"Loaded {self.n_docs} docs from {self.index_file}")
+        except Exception as e:
+            self.logger.exception(f"Failed to load index: {e}")
+            self.index_file.unlink(missing_ok=True)
+            await self.clear()
+
+    async def clear(self) -> None:
+        """Reset index to empty state and remove persisted file."""
+        self.vocab = {}
+        self.inverted_index = {}
+        self.doc_meta = {}
+        self.total_len = 0
+        self._idf_cache = {}
+        self.index_file.unlink(missing_ok=True)
+
+    async def optimize_index(self) -> None:
+        """Rebuild vocab to remove unused tokens and compact token IDs."""
+        used_token_ids: set[int] = set()
+        for tid in self.inverted_index:
+            used_token_ids.add(tid)
+        if not used_token_ids:
+            await self.clear()
+            return
+
+        # Build compact ID mapping
+        old_to_new: dict[int, int] = {}
+        new_vocab: dict[str, int] = {}
+        for token, old_tid in self.vocab.items():
+            if old_tid in used_token_ids:
+                new_tid = len(new_vocab)
+                new_vocab[token] = new_tid
+                old_to_new[old_tid] = new_tid
+
+        # Rebuild inverted index and doc_meta with new IDs
+        new_inverted_index: dict[int, dict[str, int]] = {}
+        for old_tid, postings in self.inverted_index.items():
+            new_inverted_index[old_to_new[old_tid]] = postings
+        for meta in self.doc_meta.values():
+            meta["token_ids"] = {old_to_new[t] for t in meta["token_ids"] if t in old_to_new}
+
+        self.vocab = new_vocab
+        self.inverted_index = new_inverted_index
+        self._idf_cache = {}

reme4/components/prompt_handler.py

@@ -0,0 +1,125 @@
+"""Prompt template loader and formatter with conditional-line and i18n support."""
+
+import inspect
+import json
+import re
+from pathlib import Path
+from string import Formatter
+
+import yaml
+
+# Matches a leading flag tag like "[verbose] some text".
+_FLAG_PATTERN = re.compile(r"^\[(\w+)]")
+
+
+class PromptHandler:
+    """Loads prompts from YAML/JSON or class-adjacent files and formats them.
+
+    Templates may carry a language suffix (``key_en``, ``key_zh``); ``get_prompt``
+    falls back to the bare key when no localized variant exists. ``prompt_format``
+    additionally supports per-line flags such as ``[verbose] extra text`` that
+    are kept only when the matching flag kwarg is truthy.
+    """
+
+    _SUPPORTED_EXTENSIONS = {".yaml", ".yml", ".json"}
+
+    def __init__(self, language: str = "", **kwargs):
+        # Only string entries are treated as prompts; other kwargs are ignored.
+        self.data: dict[str, str] = {k: v for k, v in kwargs.items() if isinstance(v, str)}
+        self.language: str = language.strip()
+
+    def load_prompt_by_file(
+        self,
+        prompt_file_path: str | Path | None = None,
+        overwrite: bool = True,
+    ) -> "PromptHandler":
+        """Load prompts from a YAML or JSON file; silently skip on any error."""
+        if prompt_file_path is None:
+            return self
+
+        path = Path(prompt_file_path)
+        if not path.exists() or path.suffix.lower() not in self._SUPPORTED_EXTENSIONS:
+            return self
+
+        try:
+            with path.open(encoding="utf-8") as f:
+                prompt_dict = yaml.safe_load(f) if path.suffix.lower() in (".yaml", ".yml") else json.load(f)
+        except (json.JSONDecodeError, yaml.YAMLError, OSError):
+            return self
+
+        return self.load_prompt_dict(prompt_dict, overwrite)
+
+    def load_prompt_by_class(self, cls: type, overwrite: bool = True) -> "PromptHandler":
+        """Load prompts from ``<class_module>.yaml`` (or ``.yml``) next to `cls`."""
+        try:
+            base_path = Path(inspect.getfile(cls)).with_suffix("")
+        except (TypeError, OSError):
+            return self
+
+        for ext in (".yaml", ".yml"):
+            if (prompt_path := base_path.with_suffix(ext)).exists():
+                return self.load_prompt_by_file(prompt_path, overwrite)
+
+        return self
+
+    def load_prompt_dict(self, prompt_dict: dict | None = None, overwrite: bool = True) -> "PromptHandler":
+        """Merge string entries from `prompt_dict` into the in-memory store."""
+        if not isinstance(prompt_dict, dict):
+            return self
+
+        for key, value in prompt_dict.items():
+            if isinstance(value, str) and (overwrite or key not in self.data):
+                self.data[key] = value
+
+        return self
+
+    def get_prompt(self, prompt_name: str) -> str:
+        """Return the template, preferring the language-suffixed variant when set."""
+        for key in (f"{prompt_name}_{self.language}", prompt_name) if self.language else (prompt_name,):
+            if key in self.data:
+                return self.data[key].strip()
+
+        raise KeyError(f"Prompt '{prompt_name}' not found. Available: {list(self.data.keys())[:10]}")
+
+    def has_prompt(self, prompt_name: str) -> bool:
+        """True if either the localized or bare prompt is registered."""
+        keys = (f"{prompt_name}_{self.language}", prompt_name) if self.language else (prompt_name,)
+        return any(k in self.data for k in keys)
+
+    def list_prompts(self, language_filter: str | None = None) -> list[str]:
+        """List all keys, optionally filtered to those ending with ``_<language>``."""
+        if language_filter is None:
+            return list(self.data.keys())
+        suffix = f"_{language_filter.strip()}"
+        return [k for k in self.data if k.endswith(suffix)]
+
+    def prompt_format(self, prompt_name: str, validate: bool = True, **kwargs) -> str:
+        """Render a prompt: strip inactive flag-lines, then ``str.format`` it.
+
+        Boolean kwargs are treated as flags controlling ``[flag]`` line filtering.
+        Remaining kwargs become positional substitutions for ``{var}`` placeholders.
+        With `validate=True`, missing substitutions raise ``ValueError``.
+        """
+        prompt = self.get_prompt(prompt_name)
+        flags = {k: v for k, v in kwargs.items() if isinstance(v, bool)}
+        formats = {k: v for k, v in kwargs.items() if not isinstance(v, bool)}
+
+        # Keep lines without flags; otherwise keep when at least one flag is enabled.
+        if flags:
+            lines = []
+            for line in prompt.split("\n"):
+                active_flags = _FLAG_PATTERN.findall(line)
+                cleaned = _FLAG_PATTERN.sub("", line).lstrip()
+                if not active_flags or any(flags.get(f, False) for f in active_flags):
+                    lines.append(cleaned)
+            prompt = "\n".join(lines)
+
+        if validate:
+            required = {f for _, f, _, _ in Formatter().parse(prompt) if f is not None}
+            if missing := required - set(formats.keys()):
+                raise ValueError(f"Missing format variables for '{prompt_name}': {sorted(missing)}")
+
+        return prompt.format(**formats).strip() if formats else prompt
+
+    def __repr__(self) -> str:
+        return f"PromptHandler(language='{self.language}', num_prompts={len(self.data)})"

reme4/components/runtime_context.py

@@ -0,0 +1,87 @@
+"""Per-request runtime context shared across steps and jobs."""
+
+import asyncio
+
+from ..enumeration import ChunkEnum
+from ..schema import Response, StreamChunk
+
+
+class RuntimeContext:
+    """Scratch space for a single execution.
+
+    Holds the response object, an optional stream queue, and a free-form
+    data dict accessed via mapping-style operators.
+    """
+
+    def __init__(
+        self,
+        response: Response | None = None,
+        stream_queue: asyncio.Queue | None = None,
+        **kwargs,
+    ):
+        self.response: Response = response or Response()
+        self.stream_queue: asyncio.Queue | None = stream_queue
+        self.data: dict = kwargs
+
+    def get(self, key: str, default=None):
+        """Get a value from the data dict."""
+        return self.data.get(key, default)
+
+    def update(self, data: dict) -> "RuntimeContext":
+        """Merge data into the context."""
+        self.data.update(data)
+        return self
+
+    def __getitem__(self, key: str):
+        return self.data[key]
+
+    def __setitem__(self, key: str, value):
+        self.data[key] = value
+
+    def __delitem__(self, key: str):
+        del self.data[key]
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.data
+
+    @property
+    def stream(self) -> bool:
+        """Whether streaming is enabled."""
+        return self.stream_queue is not None
+
+    @classmethod
+    def from_context(cls, context: "RuntimeContext | None" = None, **kwargs) -> "RuntimeContext":
+        """Reuse or create a RuntimeContext."""
+        # Reuse the existing context (merging kwargs) or create a new one.
+        if context is None:
+            return cls(**kwargs)
+        context.update(kwargs)
+        return context
+
+    async def _enqueue(self, chunk: StreamChunk) -> None:
+        """Put a chunk on the stream queue."""
+        if self.stream_queue is None:
+            raise RuntimeError("Stream queue not initialized")
+        await self.stream_queue.put(chunk)
+
+    async def add_stream_string(self, chunk: str, chunk_type: ChunkEnum) -> "RuntimeContext":
+        """Emit a text chunk to the stream queue."""
+        # Emit a text chunk to the stream queue.
+        await self._enqueue(StreamChunk(chunk_type=chunk_type, chunk=chunk))
+        return self
+
+    async def add_stream_done(self) -> "RuntimeContext":
+        """Emit the terminal DONE marker to close the stream."""
+        # Emit the terminal DONE marker to close the stream.
+        await self._enqueue(StreamChunk(chunk_type=ChunkEnum.DONE, chunk="", done=True))
+        return self
+
+    def apply_mapping(self, mapping: dict[str, str]) -> "RuntimeContext":
+        """Copy data[source] into data[target] for each mapping pair."""
+        # Copy data[source] into data[target] for each {source: target} pair.
+        if not mapping:
+            return self
+        for source, target in mapping.items():
+            if source in self.data:
+                self.data[target] = self.data[source]
+        return self

reme4/components/service/__init__.py

@@ -0,0 +1,11 @@
+"""Service components for exposing jobs via different protocols."""
+
+from .base_service import BaseService
+from .http_service import HttpService
+from .mcp_service import MCPService
+
+__all__ = [
+    "BaseService",
+    "HttpService",
+    "MCPService",
+]

reme4/components/service/base_service.py

@@ -0,0 +1,48 @@
+"""Base service class for exposing jobs via HTTP, MCP, etc."""
+
+from abc import abstractmethod
+from typing import TYPE_CHECKING
+
+from ..base_component import BaseComponent
+from ..job.base_job import BaseJob
+from ...enumeration import ComponentEnum
+
+if TYPE_CHECKING:
+    from ...application import Application
+
+
+class BaseService(BaseComponent):
+    """Base class for services that expose jobs via HTTP, MCP, etc."""
+
+    component_type = ComponentEnum.SERVICE
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.service = None
+
+    @abstractmethod
+    def build_service(self, app: "Application") -> None:
+        """Initialize the underlying service framework."""
+
+    @abstractmethod
+    def add_job(self, job: BaseJob) -> None:
+        """Register a single job with the service."""
+
+    @abstractmethod
+    def start_service(self, app: "Application") -> None:
+        """Start serving requests."""
+
+    def add_jobs(self, app: "Application") -> None:
+        """Register all jobs from the application context."""
+        for name, job in app.context.jobs.items():
+            try:
+                self.add_job(job)
+                self.logger.info(f"Added job: {name}")
+            except Exception as e:
+                self.logger.error(f"Failed to add job {name}: {e}")
+
+    def run_app(self, app: "Application") -> None:
+        """Build, populate, and start the service."""
+        self.build_service(app)
+        self.add_jobs(app)
+        self.start_service(app)

reme4/components/service/http_service.py

@@ -0,0 +1,99 @@
+"""HTTP service implementation for ReMe."""
+
+import asyncio
+import json
+import os
+import warnings
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
+from typing import TYPE_CHECKING
+
+import uvicorn
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import StreamingResponse
+
+from .base_service import BaseService
+from ..component_registry import R
+from ..job import BaseJob, StreamJob
+from ...constants import REME_DEFAULT_HOST, REME_DEFAULT_PORT, REME_SERVICE_INFO
+from ...schema import Request, Response
+from ...utils import execute_stream_task
+
+if TYPE_CHECKING:
+    from ...application import Application
+
+
+@R.register("http")
+class HttpService(BaseService):
+    """HTTP service: normal jobs -> JSON endpoints, stream jobs -> SSE endpoints."""
+
+    def __init__(self, host: str = REME_DEFAULT_HOST, port: int = REME_DEFAULT_PORT, **kwargs):
+        super().__init__(**kwargs)
+        self.host: str = host
+        self.port: int = port
+
+    def _add_job(self, job: BaseJob) -> None:
+        async def execute_endpoint(request: Request) -> Response:
+            return await job(**request.model_dump(exclude_none=True))
+
+        self.service.post(path=f"/{job.name}", response_model=Response, description=job.description)(execute_endpoint)
+
+    def _add_stream_job(self, job: StreamJob) -> None:
+        async def execute_stream_endpoint(request: Request) -> StreamingResponse:
+            stream_queue = asyncio.Queue()
+            task = asyncio.create_task(job(stream_queue=stream_queue, **request.model_dump(exclude_none=True)))
+
+            async def generate_stream() -> AsyncGenerator[bytes, None]:
+                async for chunk in execute_stream_task(
+                    stream_queue=stream_queue,
+                    task=task,
+                    task_name=job.name,
+                    output_format="bytes",
+                ):
+                    assert isinstance(chunk, bytes)
+                    yield chunk
+
+            return StreamingResponse(generate_stream(), media_type="text/event-stream")
+
+        self.service.post(f"/{job.name}")(execute_stream_endpoint)
+
+    def add_job(self, job: BaseJob) -> None:
+        if isinstance(job, StreamJob):
+            self._add_stream_job(job)
+        else:
+            self._add_job(job)
+
+    def build_service(self, app: "Application") -> None:
+        @asynccontextmanager
+        async def lifespan(_: FastAPI):
+            await app.start()
+            service_info = json.dumps({"host": self.host, "port": self.port})
+            os.environ[REME_SERVICE_INFO] = service_info
+            self.logger.info(f"ReMe Service started: {REME_SERVICE_INFO}={service_info}")
+            yield
+            await app.close()
+
+        self.service = FastAPI(title=app.config.app_name, lifespan=lifespan)
+        self.service.add_middleware(
+            CORSMiddleware,  # type: ignore[arg-type]
+            allow_origins=["*"],
+            allow_credentials=True,
+            allow_methods=["*"],
+            allow_headers=["*"],
+        )
+
+    def start_service(self, app: "Application") -> None:
+        # uvicorn 0.41 still imports websockets.legacy / WebSocketServerProtocol
+        # on startup; silence those specific lines since we don't use WebSocket.
+        warnings.filterwarnings(
+            "ignore",
+            category=DeprecationWarning,
+            message=r".*websockets\.legacy is deprecated.*",
+        )
+        warnings.filterwarnings(
+            "ignore",
+            category=DeprecationWarning,
+            message=r".*WebSocketServerProtocol is deprecated.*",
+        )
+        uvicorn.run(self.service, host=self.host, port=self.port, **self.kwargs)

reme4/components/service/mcp_service.py

@@ -0,0 +1,71 @@
+"""MCP (Model Context Protocol) service implementation."""
+
+import json
+import os
+from contextlib import asynccontextmanager
+from typing import TYPE_CHECKING
+
+from fastmcp import FastMCP
+from fastmcp.server.server import Transport
+from fastmcp.tools import FunctionTool
+
+from .base_service import BaseService
+from ..component_registry import R
+from ..job import StreamJob, BaseJob
+from ...constants import REME_DEFAULT_HOST, REME_DEFAULT_PORT, REME_SERVICE_INFO
+
+if TYPE_CHECKING:
+    from ...application import Application
+
+
+@R.register("mcp")
+class MCPService(BaseService):
+    """Expose jobs as MCP (Model Context Protocol) tools."""
+
+    def __init__(
+        self,
+        transport: Transport = "sse",
+        host: str = REME_DEFAULT_HOST,
+        port: int = REME_DEFAULT_PORT,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.transport: Transport = transport
+        self.host: str = host
+        self.port: int = port
+
+    def build_service(self, app: "Application") -> None:
+        @asynccontextmanager
+        async def lifespan(_: FastMCP):
+            await app.start()
+            service_info = json.dumps({"host": self.host, "port": self.port})
+            os.environ[REME_SERVICE_INFO] = service_info
+            self.logger.info(f"ReMe MCP Service started: {REME_SERVICE_INFO}={service_info}")
+            yield
+            await app.close()
+
+        self.service = FastMCP(name=app.config.app_name, lifespan=lifespan)
+
+    def add_job(self, job: "BaseJob") -> None:
+        if isinstance(job, StreamJob):
+            return
+
+        async def execute_tool(**kwargs):
+            response = await job(**kwargs)
+            return response.answer
+
+        self.service.add_tool(
+            FunctionTool(
+                name=job.name,
+                description=job.description,
+                fn=execute_tool,
+                parameters=job.parameters or None,
+            ),
+        )
+
+    def start_service(self, app: "Application") -> None:
+        transport_kwargs = {}
+        if self.transport != "stdio":
+            transport_kwargs["host"] = self.host
+            transport_kwargs["port"] = self.port
+        self.service.run(transport=self.transport, show_banner=False, **transport_kwargs)

reme4/components/tokenizer/__init__.py

@@ -0,0 +1,11 @@
+"""Tokenizer component module."""
+
+from .base_tokenizer import BaseTokenizer
+from .jieba_tokenizer import JiebaTokenizer
+from .regex_tokenizer import RegexTokenizer
+
+__all__ = [
+    "BaseTokenizer",
+    "JiebaTokenizer",
+    "RegexTokenizer",
+]

reme4/components/tokenizer/base_tokenizer.py

@@ -0,0 +1,44 @@
+"""Abstract base class for tokenizers."""
+
+from abc import abstractmethod
+from pathlib import Path
+
+import aiofiles
+
+from ..base_component import BaseComponent
+from ...enumeration import ComponentEnum
+
+
+class BaseTokenizer(BaseComponent):
+    """Base tokenizer. Subclasses must implement `tokenize`. Loads stopwords on start."""
+
+    component_type = ComponentEnum.TOKENIZER
+    DEFAULT_STOPWORDS_PATH = Path(__file__).parent / "stopwords"
+
+    def __init__(self, stopwords_path: str | Path | None = None, **kwargs):
+        super().__init__(**kwargs)
+        self.stopwords_path = Path(stopwords_path) if stopwords_path else self.DEFAULT_STOPWORDS_PATH
+        self._stopwords: set[str] = set()
+
+    async def _start(self) -> None:
+        """Load stopwords from file."""
+        if not self.stopwords_path.exists():
+            self.logger.warning(f"Stopwords file not found: {self.stopwords_path}")
+            return
+        async with aiofiles.open(self.stopwords_path, encoding="utf-8") as f:
+            content = await f.read()
+        self._stopwords = {line.strip().lower() for line in content.splitlines() if line.strip()}
+        self.logger.info(f"Loaded {len(self._stopwords)} stopwords from {self.stopwords_path}")
+
+    async def _close(self) -> None:
+        """Clear stopwords."""
+        self._stopwords.clear()
+
+    @property
+    def stopwords(self) -> set[str]:
+        """Get the loaded stopwords."""
+        return self._stopwords
+
+    @abstractmethod
+    def tokenize(self, texts: list[str], **kwargs) -> list[list[str]]:
+        """Tokenize a list of texts."""

reme4/components/tokenizer/jieba_tokenizer.py

@@ -0,0 +1,27 @@
+"""Jieba tokenizer for Chinese text segmentation."""
+
+from .base_tokenizer import BaseTokenizer
+from ..component_registry import R
+
+
+@R.register("jieba")
+class JiebaTokenizer(BaseTokenizer):
+    """Tokenizer using jieba for Chinese text segmentation."""
+
+    def __init__(self, filter_stopwords: bool = True, **kwargs):
+        super().__init__(**kwargs)
+        self.filter_stopwords = filter_stopwords
+
+    def tokenize(self, texts: list[str], lower: bool = True, **kwargs) -> list[list[str]]:
+        """Tokenize texts using jieba."""
+        import jieba
+
+        result = []
+        for text in texts:
+            tokens = jieba.cut(text)
+            if lower:
+                tokens = [x.lower() for x in tokens]
+            if self.filter_stopwords and self._stopwords:
+                tokens = [t for t in tokens if t not in self._stopwords]
+            result.append(tokens)
+        return result

reme4/components/tokenizer/regex_tokenizer.py

@@ -0,0 +1,31 @@
+"""Regex tokenizer with Chinese character splitting."""
+
+import re
+from .base_tokenizer import BaseTokenizer
+from ..component_registry import R
+
+
+@R.register("regex")
+class RegexTokenizer(BaseTokenizer):
+    """Tokenizer using regex: splits Chinese chars individually, extracts non-Chinese words."""
+
+    WORD_PATTERN = re.compile(r"(?u)\b\w\w+\b")  # 2+ char words
+    CHINESE_PATTERN = re.compile(r"[一-鿿]")  # single Chinese char
+
+    def __init__(self, filter_stopwords: bool = True, **kwargs):
+        super().__init__(**kwargs)
+        self.filter_stopwords = filter_stopwords
+
+    def tokenize(self, texts: list[str], lower: bool = True, **kwargs) -> list[list[str]]:
+        """Tokenize texts. Extracts Chinese chars, then non-Chinese words from remaining text."""
+        result = []
+        for text in texts:
+            # Extract Chinese chars individually, then non-Chinese words
+            tokens = self.CHINESE_PATTERN.findall(text)
+            tokens.extend(self.WORD_PATTERN.findall(self.CHINESE_PATTERN.sub(" ", text)))
+            if lower:
+                tokens = [t.lower() for t in tokens]
+            if self.filter_stopwords and self._stopwords:
+                tokens = [t for t in tokens if t not in self._stopwords]
+            result.append(tokens)
+        return result

reme4/config/__init__.py

@@ -0,0 +1,8 @@
+"""Config"""
+
+from .config_parser import parse_args, resolve_app_config
+
+__all__ = [
+    "parse_args",
+    "resolve_app_config",
+]

reme4/config/config_parser.py

@@ -0,0 +1,219 @@
+"""Parser for YAML config with CLI argument overrides."""
+
+import json
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+# Config files are looked up relative to this module's directory
+_CONFIG_DIR = Path(__file__).parent
+# Extensions in priority order: yaml > yml > json when stems collide
+_SUPPORTED_EXTS = (".yaml", ".yml", ".json")
+_ENV_VAR_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?}")
+# Strings like "007" / "00501" must stay as strings, not be coerced to numbers
+_LEADING_ZERO_RE = re.compile(r"^-?0\d")
+
+
+def _repl(m: re.Match) -> str:
+    name: str = m.group(1)
+    # group(2) is None when the placeholder has no `:-default` part
+    default: str | None = m.group(2)
+    v = os.environ.get(name)
+    if v is None:
+        if default is not None:
+            return default
+        raise ValueError(f"Config references undefined env var: {name}")
+    return v
+
+
+def _expand_env_vars(value: Any) -> Any:
+    """Recursively expand `${VAR}` / `${VAR:-default}` placeholders in strings."""
+    if isinstance(value, str):
+        return _ENV_VAR_RE.sub(_repl, value)
+    if isinstance(value, dict):
+        return {k: _expand_env_vars(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_expand_env_vars(v) for v in value]
+    return value
+
+
+def _discover_configs() -> dict[str, Path]:
+    """Pre-scan config directory: maps file stem (name without ext) -> Path."""
+    discovered: dict[str, Path] = {}
+    if _CONFIG_DIR.is_dir():
+        # Sort by ext priority so registration order is deterministic across filesystems
+        files = sorted(
+            (p for p in _CONFIG_DIR.iterdir() if p.is_file() and p.suffix in _SUPPORTED_EXTS),
+            key=lambda p: (_SUPPORTED_EXTS.index(p.suffix), p.name),
+        )
+        for p in files:
+            discovered.setdefault(p.stem, p)
+    return discovered
+
+
+_CONFIG_REGISTRY = _discover_configs()
+
+
+def parse_dot_notation(dot_list: list[str]) -> dict:
+    """Parse "key.subkey=value" strings into nested dict."""
+    result: dict = {}
+    for item in dot_list:
+        if "=" not in item:
+            raise ValueError(f"Invalid dot notation format (missing '='): {item}")
+        key_path, value_str = item.split("=", 1)
+        keys = key_path.split(".")
+        current = result
+        for key in keys[:-1]:
+            if key in current and not isinstance(current[key], dict):
+                raise ValueError(f"Cannot set nested key '{key_path}': '{key}' is already a value")
+            current = current.setdefault(key, {})
+        # Symmetric to the prefix check above: refuse scalar-over-dict overwrite
+        last_key = keys[-1]
+        if last_key in current and isinstance(current[last_key], dict):
+            raise ValueError(f"Cannot overwrite nested dict at '{key_path}' with scalar value")
+        current[last_key] = _convert_value(value_str)
+    return result
+
+
+def _convert_value(value_str: str) -> Any:
+    """Convert string to appropriate Python type.
+
+    Only converts "true"/"false" (case-insensitive) to boolean.
+    Use JSON format (e.g., '"yes"', '"no"') to preserve these as strings.
+    Leading-zero strings (e.g., "007", "00501") are kept as strings.
+    """
+    s = value_str.strip()
+    lower = s.lower()
+
+    # Handle special values (null, bool)
+    if lower in ("none", "null"):
+        return None
+    if lower == "true":
+        return True
+    if lower == "false":
+        return False
+
+    # Skip int/float for leading-zero strings to keep zip codes / ids intact
+    if not _LEADING_ZERO_RE.match(s):
+        for converter in (int, float):
+            try:
+                return converter(s)
+            except ValueError:
+                continue
+
+    # JSON handles lists, dicts, and explicitly-quoted strings
+    try:
+        return json.loads(s)
+    except (ValueError, json.JSONDecodeError):
+        pass
+
+    # Fallback to original string
+    return s
+
+
+def _load_config(name_or_path: str, encoding: str = "utf-8") -> dict:
+    """Load a YAML or JSON config file.
+
+    First check if name_or_path matches a pre-discovered config (key in _CONFIG_REGISTRY).
+    If not, treat as a file path and load directly.
+    """
+    # 1. Try pre-discovered configs first
+    if name_or_path in _CONFIG_REGISTRY:
+        return _read_config_file(_CONFIG_REGISTRY[name_or_path], encoding)
+
+    # 2. Treat as file path
+    p = Path(name_or_path)
+    if p.suffix in _SUPPORTED_EXTS:
+        if not p.exists():
+            raise FileNotFoundError(f"Config file not found: {p}")
+        return _read_config_file(p, encoding)
+
+    known = ", ".join(sorted(_CONFIG_REGISTRY)) if _CONFIG_REGISTRY else "none"
+    raise FileNotFoundError(f"Config file not found: {name_or_path}. Available: {known}")
+
+
+def _read_config_file(path: Path, encoding: str = "utf-8") -> dict:
+    """Read YAML or JSON file based on extension. Expands ${ENV_VAR}."""
+    with path.open(encoding=encoding) as f:
+        if path.suffix == ".json":
+            result = json.load(f)
+        else:
+            result = yaml.safe_load(f)
+    if result is None:
+        return {}
+    return _expand_env_vars(result)
+
+
+def _deep_merge(base: dict, update: dict) -> dict:
+    """Recursively merge dicts."""
+    result = base.copy()
+    for k, v in update.items():
+        if k in result and isinstance(result[k], dict) and isinstance(v, dict):
+            result[k] = _deep_merge(result[k], v)
+        else:
+            result[k] = v
+    return result
+
+
+def _strip_arg_dashes(arg: str) -> str:
+    """Strip a single leading `--` or `-` prefix (not all leading dashes)."""
+    if arg.startswith("--"):
+        return arg[2:]
+    if arg.startswith("-"):
+        return arg[1:]
+    return arg
+
+
+def parse_args(*args) -> tuple[str, dict]:
+    """Parse CLI args: first arg is action, rest are key=value pairs.
+
+    Usage: reme app config=paw.yaml service.name=test
+    Returns: (action, parsed_kv_dict)
+    """
+    if not args:
+        raise ValueError("No arguments provided")
+
+    first = _strip_arg_dashes(args[0])
+    if "=" in first:
+        raise ValueError(f"First argument must be action, got: {args[0]}")
+
+    kvs: list[str] = []
+    for raw in args[1:]:
+        arg = _strip_arg_dashes(raw)
+        if "=" in arg:
+            kvs.append(arg)
+
+    parsed = parse_dot_notation(kvs) if kvs else {}
+    return first, parsed
+
+
+def resolve_app_config(**kwargs) -> dict:
+    """Resolve full app-start config: load `config=path` file, fall back to
+    `default`, then deep-merge with the remaining kwargs as overrides.
+    """
+    from ..utils import get_logger
+
+    logger = get_logger()
+    configs: list[dict] = []
+
+    # `config=path` arrives as a string here; `config.foo=bar` arrives as a
+    # nested dict and is left in `kwargs` to be merged as a normal override.
+    config_value = kwargs.get("config")
+    if isinstance(config_value, str):
+        kwargs.pop("config")
+        logger.info(f"Loading config: {config_value}")
+        configs.append(_load_config(config_value))
+    elif "default" in _CONFIG_REGISTRY:
+        logger.info("No config specified, loading 'default'")
+        configs.append(_load_config("default"))
+
+    configs.append(kwargs)
+
+    merged: dict = {}
+    for cfg in configs:
+        merged = _deep_merge(merged, cfg)
+
+    return merged

reme4/config/default.yaml

@@ -0,0 +1,189 @@
+service:
+  backend: http
+#  backend: mcp
+
+jobs:
+  - backend: base
+    name: demo
+    description: "demo job description"
+    parameters:
+      type: object
+      properties:
+        query:
+          type: string
+          description: "query"
+        min_score:
+          type: number
+          description: "min score"
+          default: 0.5
+      required:
+        - query
+    steps:
+      - backend: demo_echo_step1
+      - backend: demo_echo_step2
+
+  - backend: base
+    name: version
+    description: "return reme4 package version"
+    parameters:
+      type: object
+      properties: {}
+    steps:
+      - backend: version_step
+
+  - backend: base
+    name: health_check
+    description: "return a concise health-check snapshot of reme4 components"
+    parameters:
+      type: object
+      properties: {}
+    steps:
+      - backend: health_check_step
+
+  - backend: base
+    name: help
+    description: "list all registered jobs with their metadata"
+    parameters:
+      type: object
+      properties: {}
+    steps:
+      - backend: help_step
+
+  - backend: base
+    name: reindex
+    description: "wipe the file store and rebuild it from the watcher's tracked files"
+    parameters:
+      type: object
+      properties: {}
+    steps:
+      - backend: reindex_step
+
+  - backend: base
+    name: search
+    description: "hybrid search over file_store: vector + keyword fused via RRF"
+    parameters:
+      type: object
+      properties:
+        query:
+          type: string
+          description: "search query"
+        limit:
+          type: integer
+          description: "max results to return"
+          default: 5
+        min_score:
+          type: number
+          description: "minimum fused score threshold (RRF scores are small; default 0 disables filter)"
+          default: 0.0
+        vector_weight:
+          type: number
+          description: "weight for vector results in [0, 1]; keyword weight = 1 - vector_weight"
+          default: 0.7
+        candidate_multiplier:
+          type: number
+          description: "candidate pool multiplier per branch (capped at 200)"
+          default: 3.0
+        expand_links:
+          type: boolean
+          description: "attach outlinks/inlinks (with neighbor meta) to each result"
+          default: true
+        max_links_per_direction:
+          type: integer
+          description: "max neighbors shown per direction per result"
+          default: 10
+      required:
+        - query
+    steps:
+      - backend: search_step
+
+  - backend: base
+    name: read
+    description: "read a markdown file (relative path under working_dir)"
+    parameters:
+      type: object
+      properties:
+        path:
+          type: string
+          description: "relative path under the working_dir (no absolute paths); markdown only"
+        start_line:
+          type: integer
+          description: "Optional, first line to read (1-based, inclusive)"
+        end_line:
+          type: integer
+          description: "Optional, last line to read (1-based, inclusive)"
+      required:
+        - path
+    steps:
+      - backend: read_step
+
+  - backend: stream
+    name: stream_demo
+    description: "stream demo job: repeat query 10x and stream char-by-char"
+    parameters:
+      type: object
+      properties:
+        query:
+          type: string
+          description: "query to echo"
+        repeat:
+          type: integer
+          description: "number of times to repeat the query"
+          default: 10
+        interval:
+          type: number
+          description: "seconds between chunks"
+          default: 0.1
+      required:
+        - query
+    steps:
+      - backend: stream_demo_step1
+      - backend: stream_demo_step2
+
+components:
+  # 1. tokenizer — no dependencies
+  tokenizer:
+    default:
+      backend: regex
+
+  # 2. embedding_model — no dependencies
+  embedding_model:
+    default:
+      backend: openai
+      model_name: text-embedding-v4
+      dimensions: 1024
+
+  # 3. file_graph — no dependencies
+  file_graph:
+    default:
+      backend: local
+
+  # 4. file_parser — no dependencies
+  file_parser:
+    default:
+      backend: default
+
+  # 5. keyword_index — depends on tokenizer
+  keyword_index:
+    default:
+      backend: bm25
+      tokenizer: default
+
+  # 6. file_store — depends on embedding_model / keyword_index / file_graph
+  file_store:
+    default:
+      backend: local
+      store_name: default
+#      embedding_model: default
+      embedding_model: ""
+      keyword_index: default
+      file_graph: default
+
+  # 7. file_watcher — depends on file_store / file_parser
+  file_watcher:
+    default:
+      backend: lite
+      watch_paths:
+        - MEMORY.md
+        - memory
+      file_store: default
+      file_parser: default

reme4/constants.py

@@ -0,0 +1,12 @@
+"""Constants"""
+
+REME_SERVICE_INFO = "REME_SERVICE_INFO"
+
+REME_DEFAULT_HOST = "127.0.0.1"
+
+REME_DEFAULT_PORT = 2333
+
+# CRUD steps: file IO limits and truncation marker (shared across CRUD steps).
+DEFAULT_MAX_BYTES = 50 * 1024
+MAX_FILE_READ_BYTES = 200 * 1024 * 1024
+TRUNCATION_NOTICE_MARKER = "<<TRUNCATION_NOTICE>>"

reme4/enumeration/__init__.py

@@ -0,0 +1,9 @@
+"""Enumeration"""
+
+from .chunk_enum import ChunkEnum
+from .component_enum import ComponentEnum
+
+__all__ = [
+    "ChunkEnum",
+    "ComponentEnum",
+]

reme4/enumeration/chunk_enum.py

@@ -0,0 +1,21 @@
+"""Chunk enumeration module."""
+
+from enum import Enum
+
+
+class ChunkEnum(str, Enum):
+    """Enumeration of possible chunk categories for stream processing."""
+
+    THINK = "think"
+
+    CONTENT = "content"
+
+    TOOL_CALL = "tool_call"
+
+    TOOL_RESULT = "tool_result"
+
+    USAGE = "usage"
+
+    ERROR = "error"
+
+    DONE = "done"