diff --git a/reme2/config/reme_todo.md b/reme2/config/reme_todo.md new file mode 100644 index 00000000..7a39e30d --- /dev/null +++ b/reme2/config/reme_todo.md @@ -0,0 +1,86 @@ +ReMe重构 +1. 根目录 + a. vault_root 改回 working dir @sen +2. file_parser + a. 抽象基类 parse: @jinli + ⅰ. 输入是path:相对路径 + ⅱ. 输出是FileMetadata & list[FileChunks] & list[FileEdge] + b. default parser 兼容老方案 @jinli + ⅰ. 带overlap的chunking策略 ,不输出FileEdge + c. markdown parser @sen + ⅰ. 根据markdown ast做chunk,不需要overlap + ⅱ. 增加一个索引的chunk chunk_type @锦鲤 file_chunk_type content/index + ⅲ. 增加link的正则解析:predicate:: [[path#anchor]] +3. file_store @sen + a. 抽象存储: + ⅰ. filenode = file + path + st_mtime + metadata + list[FileEdge] + ⅱ. graph=dict[str, filenode] 内存+json + ⅲ. list[FileChunk] 存db + b. 抽象基类 + ⅰ. graph:fellow dict的操作 update/get/set + ⅱ. chunks dict[str, list[chunk]] + 1. delete_chunks_by_path + 2. update_chunks_by_path + 3. list_chunks_by_path + 4. vector_search/keyword_search + ⅲ. 手写一个bm25检索 + ⅳ. 【核心】检索机制 vector bm25 graph 如何进行融合 +4. file_watcher @jinli + a. 抽象基类 + ⅰ. on_start: + 1. file_store 的start 在前,加载graph,file_watcher在后,递归扫描目录 + a. 通过ms_time对比graph,on_change 进行改动 + ⅱ. on_change: + 1. 更新/增加: + a. delete_chunks_by_path 更新数据库 + b. upate_chunks_by_path 更新数据库 + c. 更新graph + 2. 删除 + a. delete_chunks_by_path 更新数据库 + +MemorySchema +1. markdown文件结构 @sen + a. formatter: + ⅰ. title + ⅱ. desc + ⅲ. tags + ⅳ. +2. memory文件结构目录 + a. MEMORY.md + b. msg/files -> daily/YYYYMMDD/YYYYMMDD.md + xxxx.md + ⅰ. YYYYMMDD.md + 1. xxx -> xxxx.md + 2. xxx -> xxxd.md + ⅱ. + c. daily -> topic/topic_l1/topic_l1.md + xxx.md + topic_l2 + d. proactive + +steps: +1. 治理(算法+LLM): + a. 节点关联P0:现有的链接做补充,挖掘新的LLM的link + ⅰ. /Users/yuli/workspace/ReMe/reme2/component/edge_extractor/llm_edge_extractor.py + ⅱ. 移动到steps + b. 节点整合/节点拆分/节点归档 + c. 健康度检查 +2. retrieve 调用store的检索 +3. 原子steps:reme edit +4. 组合steps:总结: + a. - freq (every_n_turn、compact) -> daily_summarizer + b. topic (/dream ) -> topic_summarizer(daily_xx -> topic_xx) + c. proactive -> proactive_summarizer(personal_xxx -> proactive_query - pre_query + + + + + + + + + + + + + + + +Qwenpaw \ No newline at end of file diff --git a/reme2/memory/schema/memory.py b/reme2/memory/schema/memory.py index ce219c66..213172be 100644 --- a/reme2/memory/schema/memory.py +++ b/reme2/memory/schema/memory.py @@ -213,8 +213,6 @@ class Memory(BaseModel): title: str = Field(default="") description: str = Field(default="") tags: list[str] = Field(default_factory=list) - created: date | None = None - updated: date | None = None # -- The four behavioral axes ------------------------------------------ @@ -225,9 +223,6 @@ class Memory(BaseModel): # -- Cross-cutting graph fields ---------------------------------------- - topics: list[str] = Field(default_factory=list, description="Outbound wikilinks to class memories.") - parent: str | None = Field(default=None, description="Wikilink to owning memory (e.g. material → event).") - # -- Role / lifecycle / source-conditional fields ---------------------- confidence: Confidence | None = Field( @@ -238,11 +233,6 @@ class Memory(BaseModel): default=None, description="Lifecycle state. Meaningful only when lifecycle == streaming.", ) - origin_session_id: str | None = Field( - default=None, - alias="originSessionId", - description="Capture session id. Set when source == auto.", - ) # -- Migration: pre-validate hook -------------------------------------- diff --git a/reme2/schema/file_chunk.py b/reme2/schema/file_chunk.py index 87d1a17a..5c29a1bd 100644 --- a/reme2/schema/file_chunk.py +++ b/reme2/schema/file_chunk.py @@ -9,6 +9,7 @@ class FileChunk(BaseNode): path: str = Field(...) start_line: int = Field(...) end_line: int = Field(...) + hash: str = Field(...) scores: dict[str, float] = Field(default_factory=dict) diff --git a/reme2/schema/file_metadata.py b/reme2/schema/file_metadata.py deleted file mode 100644 index 828eee58..00000000 --- a/reme2/schema/file_metadata.py +++ /dev/null @@ -1,8 +0,0 @@ -from pydantic import BaseModel, Field - - -class FileMetadata(BaseModel): - file: str = Field(...) - path: str = Field(...) - st_mtime: float = Field(...) - metadata: dict = Field(default_factory=dict) diff --git a/reme2/schema/file_node.py b/reme2/schema/file_node.py new file mode 100644 index 00000000..303648e8 --- /dev/null +++ b/reme2/schema/file_node.py @@ -0,0 +1,28 @@ +from pydantic import BaseModel, Field, ConfigDict + +from reme.core.schema import FileMetadata + + +class FileNode(BaseModel): + model_config = ConfigDict(extra="allow") + + path: str = Field(...) + st_mtime: float = Field(...) + edges: list[FileEdge] = Field(default_factory=list) + + @property + def file(self): + ... + + + +class FileMetadata(BaseModel): + # FileNode -> FileMetadata + path: str = Field(...) + + edges: list[FileEdge] = Field(default_factory=list) + + title: str = Field(default="") + description: str = Field(default="") + tags: list[str] = Field(default_factory=list) +