From 3367d48ea1aca3cfc71989bc9c3526eecfbff9eb Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Sat, 16 May 2026 22:13:25 +0800 Subject: [PATCH] up --- docs4/expert.yaml | 378 ------------------ docs4/{obsidian.md => reme_design.md} | 66 +++ docs4/reme_todo.md | 86 ---- docs4/service.yaml | 270 ------------- docs4/todo.md | 2 + .../file_watcher/base_file_watcher.py | 43 +- .../file_watcher/lite_file_watcher.py | 75 ++-- reme4/components/keyword_index/bm25_index.py | 2 + reme4/config/default.yaml | 69 +++- reme4/steps/__init__.py | 5 +- reme4/steps/demo.py | 53 +++ reme4/steps/demo/__init__.py | 19 - 12 files changed, 252 insertions(+), 816 deletions(-) delete mode 100644 docs4/expert.yaml rename docs4/{obsidian.md => reme_design.md} (57%) delete mode 100644 docs4/reme_todo.md delete mode 100644 docs4/service.yaml create mode 100644 docs4/todo.md create mode 100644 reme4/steps/demo.py delete mode 100644 reme4/steps/demo/__init__.py diff --git a/docs4/expert.yaml b/docs4/expert.yaml deleted file mode 100644 index 1cb2e0e0..00000000 --- a/docs4/expert.yaml +++ /dev/null @@ -1,378 +0,0 @@ -app_name: reme-expert -enable_logo: false -log_to_console: true -log_to_file: false - -# Expert-exposure profile: every memory_* read + write primitive plus -# `sync` (event log) are surfaced as MCP tools. The agent picks whatever -# it needs — finest granularity, no opinion enforced beyond the -# wikilink-uniqueness gate. -# -# **No `ingest` tool.** Cold-path R-M-W in expert mode is owned by the -# host agent (Claude Code, with the `reme-expert` SKILL loaded), not by -# reme2's internal ReActAgent. The reme-expert plugin spawns a -# `reme-distiller` subagent at SessionEnd to handle distillation in its -# own context window, using the same memory_* primitives. -# -# Use when you want the agent to manage memory directly with full -# control. For an opinionated minimal surface (where reme2's internal -# Ingestor owns R-M-W), see ./service.yaml. - -service: - backend: mcp - transport: stdio - sidecar_http: true - sidecar_http_host: "127.0.0.1" - sidecar_http_port: 8765 - sidecar_info_path: "./vault/.reme/sidecar.json" - -jobs: - # -- Write entry points ------------------------------------------------ - - - backend: base - name: sync - description: | - Hot-path event sync: idempotent upsert of an event FOLDER under - `events/{date}/{name}/`. The folder contains the index `{name}.md` - (Event schema, frontmatter + narrative + Materials footer) plus - any raw materials you pass — conversation snippets, tool outputs, - data dumps. The watcher indexes everything inside. - - CONTINUITY MODEL: pick a stable `name` per logical thread and - call `sync` repeatedly through the task. Each call extends the - same folder: - * new `content` → appended under a `## Update — {iso}` section - * new `materials` → siblings (auto-suffix on filename collision) - * `topics` + `tags` merged (union) into frontmatter - * Materials footer regenerated to list every artifact in the folder - - First call (folder doesn't exist) → CREATE; subsequent calls with - the same `name` while the event is `status: active` → APPEND. - If the event is `status: distilled` / `archived`, `sync` REFUSES - and returns a `suggested_name` so you start a fresh thread - instead of mutating prior cognition. - - Zero LLM cost. Call CONTINUOUSLY through a task as facts land, - and especially at PreCompact to dump verbose raw text into - `materials` before context truncation loses it. The folder is - the unit `ingest` later reads from. - parameters: - type: object - required: [name] - properties: - name: - type: string - description: "kebab-case event identifier (folder + index stem). Reuse the same name across calls in one thread to keep extending the same folder." - description: { type: string, description: "one-line summary for index frontmatter (set on initial create only)" } - content: { type: string, description: "markdown body. Initial create: the body. Subsequent calls: appended as a `## Update — {iso}` section." } - topics: - type: array - items: { type: string } - description: "related topic wikilinks ('[[X]]' or '[[topics/X/X]]'). Unioned into frontmatter on append." - tags: { type: array, items: { type: string }, description: "free-form tags; unioned on append." } - materials: - type: array - description: "Raw artifacts written as siblings of the index inside the event folder. Filenames must be safe (letters/digits/dot/underscore/dash). On filename collision with an existing artifact, auto-suffixes (foo.txt → foo-2.txt)." - items: - type: object - required: [filename, content] - properties: - filename: { type: string, description: "e.g. 'raw-prompt.md', 'tool-output.txt', 'snapshot.json'" } - content: { type: string } - on_date: { type: string, description: "ISO date for events/{date}/ bucket; defaults to today" } - origin_session_id: { type: string, description: "set on initial create only" } - steps: - - backend: sync - - # -- Read tools -------------------------------------------------------- - - - backend: base - name: memory_search - description: "Hybrid (vector + keyword) search over chunks." - parameters: - type: object - required: [query] - properties: - query: { type: string } - max_results: { type: integer, default: 5 } - min_score: { type: number, default: 0.1 } - paths: { type: array, items: { type: string } } - tags: { type: array, items: { type: string } } - exclude_paths: { type: array, items: { type: string } } - steps: - - backend: memory_search - - - backend: base - name: memory_graph_search - description: | - Three-way fusion search: vector + keyword + graph (BFS over wikilinks). - Pulls in chunks reachable through linked topics/events that pure - relevance search would miss. - parameters: - type: object - properties: - query: { type: string } - max_results: { type: integer, default: 5 } - min_score: { type: number, default: 0.0 } - graph_depth: { type: integer, default: 1 } - seeds: { type: array, items: { type: string } } - paths: { type: array, items: { type: string } } - tags: { type: array, items: { type: string } } - exclude_paths: { type: array, items: { type: string } } - steps: - - backend: memory_graph_search - - - backend: base - name: memory_get - description: "Read frontmatter + body of a single file." - parameters: - type: object - required: [path] - properties: - path: { type: string } - include_chunks: { type: boolean, default: false } - steps: - - backend: memory_get - - - backend: base - name: memory_list - description: | - List indexed files filtered by frontmatter exact-match, tags, or - path prefix. Returns {items: [{path, metadata}], count}. - parameters: - type: object - properties: - metadata: { type: object } - tags: { type: array, items: { type: string } } - path_prefix: { type: string } - limit: { type: integer, default: 100 } - steps: - - backend: memory_list - - - backend: base - name: memory_backlinks - description: "Files linking TO the given path (with edge predicates)." - parameters: - type: object - required: [path] - properties: - path: { type: string } - steps: - - backend: memory_backlinks - - - backend: base - name: memory_links - description: "Files the given path links to (resolved, with edge predicates)." - parameters: - type: object - required: [path] - properties: - path: { type: string } - steps: - - backend: memory_links - - - backend: base - name: memory_resolve_wikilink - description: | - Resolve a `[[target]]` wikilink to a vault path. Stem-form (`X`) - consults the file_store's stem index; path-form (`a/b` or `a/b.md`) - is anchored at the vault root. - parameters: - type: object - required: [wikilink] - properties: - wikilink: { type: string } - steps: - - backend: memory_resolve_wikilink - - - backend: base - name: memory_count_tokens - description: "Estimate token count for a file body or raw text." - parameters: - type: object - properties: - path: { type: string } - text: { type: string } - steps: - - backend: memory_count_tokens - - - backend: base - name: memory_lint - description: | - Read-only projection of the Maintainer's lint findings. Walks the - indexed files (optionally restricted to a path prefix) and returns - structural issues — broken wikilinks, schema violations, stem - collisions. Never mutates. The agent decides what to do with each - finding using existing memory_* primitives (rename, update, - property_update, archive, delete). - parameters: - type: object - properties: - target_prefix: - type: string - description: "restrict scan to relpaths starting with this prefix (e.g. 'events/2026-05-09/'). Empty string scans the whole vault." - steps: - - backend: memory_lint - - # -- Write primitives (raw building blocks) ---------------------------- - - - backend: base - name: memory_create - description: | - Create a new file (raw primitive — no LLM reasoning). Prefer - `ingest` when you want an LLM curator. Two gates run unless - `force=true`: (1) path template — must be `topics/{folder}/{name}.md`, - `events/{date}/{name}/...`, or `Archive/...`; (2) wikilink uniqueness - — `[[stem]]` must resolve to ≤1 path post-create. - parameters: - type: object - required: [path] - properties: - path: { type: string } - metadata: { type: object } - content: { type: string } - overwrite: { type: boolean, default: false } - force: - type: boolean - default: false - description: "bypass BOTH the path-template gate and the wikilink-uniqueness gate. Use only when you intentionally need a non-template path or accept the ambiguity." - steps: - - backend: memory_create - - - backend: base - name: memory_update - description: | - Edit-style content update: replace `old_string` with `new_string` - in the file body. For frontmatter changes use `memory_property_update`. - parameters: - type: object - required: [path, old_string, new_string] - properties: - path: { type: string } - old_string: { type: string } - new_string: { type: string } - replace_all: { type: boolean, default: false } - steps: - - backend: memory_update - - - backend: base - name: memory_property_update - description: | - Update a single YAML frontmatter key. value=null deletes the key. - When key='status', enforces the active → distilled → archived - single-direction state machine (skip / reverse refused). Pass - `force=true` to bypass. - parameters: - type: object - required: [path, key] - properties: - path: { type: string } - key: { type: string } - value: {} - force: - type: boolean - default: false - description: "bypass the status state machine (only meaningful when key='status')." - steps: - - backend: memory_property_update - - - backend: base - name: memory_rename - description: | - Rename a file and rewrite all incoming `[[wikilink]]` references - across the vault. Refuses on destination conflict or stem ambiguity. - parameters: - type: object - required: [old_path, new_path] - properties: - old_path: { type: string } - new_path: { type: string } - steps: - - backend: memory_rename - - - backend: base - name: memory_delete - description: "Delete a file." - parameters: - type: object - required: [path] - properties: - path: { type: string } - steps: - - backend: memory_delete - - - backend: base - name: memory_archive - description: | - Archive a file: flip `status: archived` then move under - `//`. - parameters: - type: object - required: [path] - properties: - path: { type: string } - archive_dir: { type: string, default: "Archive" } - steps: - - backend: memory_archive - -components: - # Ingestor LLM (opt-in). Without this the Ingestor degrades to a - # direct create from explicit `target_path`; edits/renames/deletes - # require the LLM. Uncomment + provide LLM_API_KEY to enable. - # - # as_llm: - # default: - # backend: openai - # model_name: ${LLM_MODEL_NAME:-gpt-4o-mini} - # api_key: ${LLM_API_KEY} - # client_kwargs: - # base_url: ${LLM_BASE_URL:-https://api.openai.com/v1} - # stream: false - # - # as_llm_formatter: - # default: - # backend: openai - - as_token_counter: - default: - backend: estimated - - # Embedding is opt-in: leave embedding_model="" on file_store to run - # keyword-only; uncomment + flip to "default" to enable hybrid search. - # - # embedding_model: - # default: - # backend: openai - # model_name: ${EMBEDDING_MODEL_NAME:-text-embedding-3-small} - # dimensions: 1536 - # pass_dimensions: false - # enable_cache: true - # max_batch_size: 10 - # max_cache_size: 2000 - # max_input_length: 8192 - - file_parser: - md: - backend: md - default: - backend: text - - file_store: - default: - backend: local - embedding_model: "" - store_name: "reme" - db_path: "./vault/.reme" - working_dir: "./vault" - - file_watcher: - default: - backend: full - file_store: default - default_parser: md - recursive: true - - # Retriever (`hybrid`) is a Step, not a pre-instantiated component — - # the `memory_search` / `memory_graph_search` shells build it on - # demand. To tune defaults, pass knobs (`vector_weight`, `graph_*`, - # …) on the step config under each job below. diff --git a/docs4/obsidian.md b/docs4/reme_design.md similarity index 57% rename from docs4/obsidian.md rename to docs4/reme_design.md index 4d342f53..46ec739a 100644 --- a/docs4/obsidian.md +++ b/docs4/reme_design.md @@ -48,4 +48,70 @@ +2. file_parser + a. 抽象基类 parse: @jinli + ⅰ. 输入是path：相对路径 + ⅱ. 输出是FileMetadata & list[FileChunks] & list[FileEdge] + b. default parser 兼容老方案 @jinli + ⅰ. 带overlap的chunking策略 ，不输出FileEdge + c. markdown parser @sen + ⅰ. 根据markdown ast做chunk，不需要overlap + ⅱ. 增加一个索引的chunk chunk_type @锦鲤 file_chunk_type content/index + ⅲ. 增加link的正则解析：predicate:: [[path#anchor]] +3. file_store @sen + a. 抽象存储： + ⅰ. filenode = file + path + st_mtime + metadata + list[FileEdge] + ⅱ. graph=dict[str, filenode] 内存+json + ⅲ. list[FileChunk] 存db + b. 抽象基类 + ⅰ. graph：fellow dict的操作 update/get/set + ⅱ. chunks dict[str, list[chunk]] + 1. delete_chunks_by_path + 2. update_chunks_by_path + 3. list_chunks_by_path + 4. vector_search/keyword_search + ⅲ. 手写一个bm25检索 + ⅳ. 【核心】检索机制 vector bm25 graph 如何进行融合 +4. file_watcher @jinli + a. 抽象基类 + ⅰ. on_start: + 1. file_store 的start 在前，加载graph，file_watcher在后，递归扫描目录 + a. 通过ms_time对比graph，on_change 进行改动 + ⅱ. on_change: + 1. 更新/增加: + a. delete_chunks_by_path 更新数据库 + b. upate_chunks_by_path 更新数据库 + c. 更新graph + 2. 删除 + a. delete_chunks_by_path 更新数据库 +MemorySchema +1. markdown文件结构 @sen + a. formatter： + ⅰ. title + ⅱ. desc + ⅲ. tags + ⅳ. +2. memory文件结构目录 + a. MEMORY.md + b. msg/files -> daily/YYYYMMDD/YYYYMMDD.md + xxxx.md + ⅰ. YYYYMMDD.md + 1. xxx -> xxxx.md + 2. xxx -> xxxd.md + ⅱ. + c. daily -> topic/topic_l1/topic_l1.md + xxx.md + topic_l2 + d. proactive + +steps: +1. 治理（算法+LLM）： + a. 节点关联P0：现有的链接做补充，挖掘新的LLM的link + ⅰ. /Users/yuli/workspace/ReMe/reme2/component/edge_extractor/llm_edge_extractor.py + ⅱ. 移动到steps + b. 节点整合/节点拆分/节点归档 + c. 健康度检查 +2. retrieve 调用store的检索 +3. 原子steps：reme edit +4. 组合steps：总结： + a. - freq (every_n_turn、compact) -> daily_summarizer + b. topic (/dream ) -> topic_summarizer(daily_xx -> topic_xx) + c. proactive -> proactive_summarizer(personal_xxx -> proactive_query - pre_query diff --git a/docs4/reme_todo.md b/docs4/reme_todo.md deleted file mode 100644 index a3fecfa6..00000000 --- a/docs4/reme_todo.md +++ /dev/null @@ -1,86 +0,0 @@ -ReMe重构 -1. 根目录 - a. ✅ vault_root → working_dir @sen -2. file_parser - a. 抽象基类 parse: @jinli - ⅰ. 输入是path：相对路径 - ⅱ. 输出是FileMetadata & list[FileChunks] & list[FileEdge] - b. default parser 兼容老方案 @jinli - ⅰ. 带overlap的chunking策略 ，不输出FileEdge - c. markdown parser @sen - ⅰ. 根据markdown ast做chunk，不需要overlap - ⅱ. 增加一个索引的chunk chunk_type @锦鲤 file_chunk_type content/index - ⅲ. 增加link的正则解析：predicate:: [[path#anchor]] -3. file_store @sen - a. 抽象存储： - ⅰ. filenode = file + path + st_mtime + metadata + list[FileEdge] - ⅱ. graph=dict[str, filenode] 内存+json - ⅲ. list[FileChunk] 存db - b. 抽象基类 - ⅰ. graph：fellow dict的操作 update/get/set - ⅱ. chunks dict[str, list[chunk]] - 1. delete_chunks_by_path - 2. update_chunks_by_path - 3. list_chunks_by_path - 4. vector_search/keyword_search - ⅲ. 手写一个bm25检索 - ⅳ. 【核心】检索机制 vector bm25 graph 如何进行融合 -4. file_watcher @jinli - a. 抽象基类 - ⅰ. on_start: - 1. file_store 的start 在前，加载graph，file_watcher在后，递归扫描目录 - a. 通过ms_time对比graph，on_change 进行改动 - ⅱ. on_change: - 1. 更新/增加: - a. delete_chunks_by_path 更新数据库 - b. upate_chunks_by_path 更新数据库 - c. 更新graph - 2. 删除 - a. delete_chunks_by_path 更新数据库 - -MemorySchema -1. markdown文件结构 @sen - a. formatter： - ⅰ. title - ⅱ. desc - ⅲ. tags - ⅳ. -2. memory文件结构目录 - a. MEMORY.md - b. msg/files -> daily/YYYYMMDD/YYYYMMDD.md + xxxx.md - ⅰ. YYYYMMDD.md - 1. xxx -> xxxx.md - 2. xxx -> xxxd.md - ⅱ. - c. daily -> topic/topic_l1/topic_l1.md + xxx.md + topic_l2 - d. proactive - -steps: -1. 治理（算法+LLM）： - a. 节点关联P0：现有的链接做补充，挖掘新的LLM的link - ⅰ. /Users/yuli/workspace/ReMe/reme2/component/edge_extractor/llm_edge_extractor.py - ⅱ. 移动到steps - b. 节点整合/节点拆分/节点归档 - c. 健康度检查 -2. retrieve 调用store的检索 -3. 原子steps：reme edit -4. 组合steps：总结： - a. - freq (every_n_turn、compact) -> daily_summarizer - b. topic (/dream ) -> topic_summarizer(daily_xx -> topic_xx) - c. proactive -> proactive_summarizer(personal_xxx -> proactive_query - pre_query - - - - - - - - - - - - - - - -Qwenpaw \ No newline at end of file diff --git a/docs4/service.yaml b/docs4/service.yaml deleted file mode 100644 index bf2e3fe6..00000000 --- a/docs4/service.yaml +++ /dev/null @@ -1,270 +0,0 @@ -app_name: reme-service -enable_logo: false -log_to_console: true -log_to_file: false - -# Service profile: service-aligned MCP surface — three tools, one per -# memory service: -# `retrieve` — graph-aware hybrid retrieval (Retriever projection) -# `remember` — single write entry point (Ingestor projection) -# * mode=log → zero-LLM event-folder upsert -# * mode=distill → LLM R-M-W into topic graph -# `maintain` — vault hygiene sweep (Maintainer projection) -# runs lint + decay; merge/split require an LLM and -# are off by default. -# -# The agent calls `remember(mode=log, name=…, content=…, materials=…)` -# continuously through the task — picking a stable `name` per logical -# thread so each call extends the same event folder rather than -# fragmenting. At task completion (or PreCompact / SessionEnd) it -# calls `remember(mode=distill, content=…, related_paths=…)` once to -# distill the active events into topic-level cognition. `maintain` is -# typically cron-driven, but the agent can invoke it explicitly when -# it suspects vault drift. -# -# For full direct control over every memory_* primitive (raw writes, -# fine-grained reads, separate sync/ingest tools, memory_lint) see -# ./expert.yaml. - -service: - backend: mcp - transport: stdio - sidecar_http: true - sidecar_http_host: "127.0.0.1" - sidecar_http_port: 8765 - sidecar_info_path: "./vault/.reme/sidecar.json" - -jobs: - - backend: base - name: retrieve - description: | - Graph-aware hybrid retrieval: vector + keyword + 1-hop wikilink - BFS fusion. Use for "what do I know about X" / "did I work on Y" / - "what's connected to [[Z]]". Anchor mode: include `[[Target]]` in - the query to seed BFS at that file. Topic-rooted mode: pass `seeds` - explicitly. Returns chunks ranked by combined relevance + graph - proximity, each tagged with `graph_hop`. - parameters: - type: object - properties: - query: { type: string } - max_results: { type: integer, default: 5 } - min_score: { type: number, default: 0.0 } - graph_depth: - type: integer - default: 1 - description: "BFS hops from seeds. 1 covers immediate neighbors." - seeds: - type: array - items: { type: string } - description: "Explicit seed paths (topic-rooted mode)." - paths: { type: array, items: { type: string } } - tags: { type: array, items: { type: string } } - exclude_paths: { type: array, items: { type: string } } - steps: - - backend: memory_graph_search - - - backend: base - name: remember - description: | - Single write entry point — projects the Ingestor service. Two - modes via the `mode` parameter: - - * `mode: log` (zero LLM, hot path) — idempotent upsert of an - event FOLDER under `events/{date}/{name}/`. The folder - contains the index `{name}.md` (Event schema, frontmatter + - narrative + Materials footer) plus any raw materials you - pass — conversation snippets, tool outputs, data dumps. The - watcher indexes everything inside. - - CONTINUITY MODEL: pick a stable `name` per logical thread - and call `remember(mode=log, ...)` repeatedly through the - task. Each call extends the same folder: - - new `content` → appended under a `## Update — {iso}` - section - - new `materials` → siblings (auto-suffix on filename - collision) - - `topics` + `tags` merged (union) into frontmatter - - Materials footer regenerated to list every artifact - - First call (folder doesn't exist) → CREATE; subsequent calls - with the same `name` while the event is `status: active` → - APPEND. If `status: distilled` / `archived`, REFUSES and - returns `suggested_name` so you start a fresh thread instead - of mutating prior cognition. Call CONTINUOUSLY through a task - as facts land, especially at PreCompact to dump verbose raw - text into `materials` before context truncation. - - * `mode: distill` (LLM R-M-W loop, cold path) — DEFAULT. Run - on EXPLICIT HANDOFF only: task completion / SessionEnd / when - you decide the working set is ready. Not a per-turn tool. - - Hand off the working set in two interchangeable forms: - - `content` — inline material to distill (hint, summary, or - raw text) - - `related_paths` — pointers (event folder indexes; the - Ingestor follows `## Materials` to read each artifact, - individual material files, candidate topics). - - The Ingestor reads the working set + linked topics, decides - which existing topics to update / create, and flips each - distilled event's status to "distilled". Returns an audit - trail. - parameters: - type: object - required: [content] - properties: - mode: - type: string - enum: [log, distill] - default: distill - description: "log = zero-LLM event-folder upsert (requires `name`); distill = LLM R-M-W into topic graph (default)." - # mode=log params - name: - type: string - description: "(mode=log) kebab-case event identifier (folder + index stem). Reuse the same name across calls in one thread to keep extending the same folder." - description: - type: string - description: "(mode=log) one-line summary for index frontmatter (set on initial create only)." - topics: - type: array - items: { type: string } - description: "(mode=log) related topic wikilinks. Unioned into frontmatter on append." - tags: - type: array - items: { type: string } - description: "(mode=log) free-form tags; unioned on append." - materials: - type: array - description: "(mode=log) raw artifacts written as siblings of the index. Filenames must be safe (letters/digits/dot/underscore/dash). Filename collision auto-suffixes (foo.txt → foo-2.txt)." - items: - type: object - required: [filename, content] - properties: - filename: { type: string, description: "e.g. 'raw-prompt.md', 'tool-output.txt'" } - content: { type: string } - on_date: - type: string - description: "(mode=log) ISO date for events/{date}/ bucket; defaults to today." - origin_session_id: - type: string - description: "(mode=log) optional source session identifier (set on initial create only)." - # shared / mode=distill params - content: - type: string - description: "Required. mode=log: markdown body for the index (initial create) or appended `## Update — {iso}` section. mode=distill: inline material the Ingestor distills — hint, summary, or raw text." - hint: - type: string - description: "(mode=distill) caller guidance about target / intent." - target_path: - type: string - description: "(mode=distill) optional suggested path; required for the no-LLM degraded path." - metadata: - type: object - description: "(mode=distill) suggested frontmatter for any new topic." - related_paths: - type: array - items: { type: string } - description: "(mode=distill) pointers — event folder indexes (Ingestor reads materials from `## Materials`), individual material files, or candidate topics." - steps: - - backend: ingestor - - - backend: base - name: maintain - description: | - Vault hygiene sweep — projects the Maintainer service. One pass: - scan signals → propose ops → resolve conflicts → apply. Returns - an audit trail of what ran and what changed. - - Default behavior runs `lint` (broken wikilinks, schema violations, - stem collisions — read-only diagnostics) and `decay` (move stale - events past their freshness window under `/Archive/`). - Merge / split require an LLM and are off unless explicitly opted - into via `ops`. - - Typically cron-driven; the agent invokes it on demand when it - suspects vault drift (after a heavy session of edits, after a - bulk rename, etc.). Dry-run by default — flip `dry_run=false` to - apply changes. - parameters: - type: object - properties: - target_prefix: - type: string - description: "restrict scan to relpaths starting with this prefix (e.g. 'events/2026-05-09/'). Empty string scans the whole vault." - dry_run: - type: boolean - default: true - description: "if true (default) returns the plan without mutating; flip to false to actually apply lint+decay (and merge/split if opted in)." - ops: - type: array - items: - type: string - enum: [lint, decay, merge, split] - description: "subset of ops to run. Defaults to ['lint','decay']. Merge/split are LLM-driven and currently scaffolded — enabling them without an LLM is a no-op." - decay_days: - type: integer - description: "freshness window for the decay proposer (default = Maintainer constructor `decay_days`, typically 90)." - steps: - - backend: maintainer - -components: - # Ingestor LLM (opt-in). Without this the Ingestor degrades to a - # direct create from explicit `target_path`; edits/renames/deletes - # require the LLM. Uncomment + provide LLM_API_KEY to enable. - # - # as_llm: - # default: - # backend: openai - # model_name: ${LLM_MODEL_NAME:-gpt-4o-mini} - # api_key: ${LLM_API_KEY} - # client_kwargs: - # base_url: ${LLM_BASE_URL:-https://api.openai.com/v1} - # stream: false - # - # as_llm_formatter: - # default: - # backend: openai - - as_token_counter: - default: - backend: estimated - - # Embedding is opt-in: leave embedding_model="" on file_store to run - # keyword-only; uncomment + flip to "default" to enable hybrid search. - # - # embedding_model: - # default: - # backend: openai - # model_name: ${EMBEDDING_MODEL_NAME:-text-embedding-3-small} - # dimensions: 1536 - # pass_dimensions: false - # enable_cache: true - # max_batch_size: 10 - # max_cache_size: 2000 - # max_input_length: 8192 - - file_parser: - md: - backend: md - default: - backend: text - - file_store: - default: - backend: local - embedding_model: "" - store_name: "reme" - db_path: "./vault/.reme" - working_dir: "./vault" - - file_watcher: - default: - backend: full - file_store: default - default_parser: md - recursive: true - - # Retriever (`hybrid`) is a Step, not a pre-instantiated component — - # the `query` shell builds it on demand. Tune defaults by attaching - # knobs to the `memory_graph_search` step under the `query` job above. diff --git a/docs4/todo.md b/docs4/todo.md new file mode 100644 index 00000000..ef3a474b --- /dev/null +++ b/docs4/todo.md @@ -0,0 +1,2 @@ +1. 完善mcp_servers config +2. 完善mcp/http的服务测试 \ No newline at end of file diff --git a/reme4/components/file_watcher/base_file_watcher.py b/reme4/components/file_watcher/base_file_watcher.py index 236b06a9..dfee23d7 100644 --- a/reme4/components/file_watcher/base_file_watcher.py +++ b/reme4/components/file_watcher/base_file_watcher.py @@ -69,18 +69,29 @@ class BaseFileWatcher(BaseComponent): return True return any(path.endswith("." + s.strip(".")) for s in self.suffix_filters) - async def scan_existing_files(self) -> list[Path]: - """Collect all watchable files under watch_paths.""" - files: list[Path] = [] + def _get_relative_path(self, path: str | Path) -> str: + """Return path relative to working_dir, or absolute path if outside.""" + file_path = Path(path).absolute() + try: + return str(file_path.relative_to(self.working_path.absolute())) + except ValueError: + return str(file_path) + + def _get_absolute_path(self, path: str | Path) -> Path: + """Return absolute path; relative paths are resolved against working_dir.""" + p = Path(path) + return p if p.is_absolute() else self.working_path / p + + async def scan_existing_files(self) -> dict[str, Path]: + """Collect watchable files under watch_paths as {relative_path: absolute_path}.""" + files: dict[str, Path] = {} for path in self.watch_paths: if not path.exists(): continue - if path.is_file(): - if self.watch_filter(Change.added, str(path)): - files.append(path) - else: - items = path.rglob("*") if self.recursive else path.iterdir() - files.extend(p for p in items if p.is_file() and self.watch_filter(Change.added, str(p))) + candidates = [path] if path.is_file() else (path.rglob("*") if self.recursive else path.iterdir()) + for p in candidates: + if p.is_file() and self.watch_filter(Change.added, str(p)): + files[self._get_relative_path(p)] = p.absolute() return files async def clear_store(self): @@ -94,7 +105,7 @@ class BaseFileWatcher(BaseComponent): if self.file_store is None: raise ValueError("file_store is not initialized!") await self.file_store.clear() - await self.on_added(await self.scan_existing_files()) + await self.on_added(list((await self.scan_existing_files()).keys())) @abstractmethod async def watch_loop(self): @@ -105,13 +116,13 @@ class BaseFileWatcher(BaseComponent): """Sync the store with the current state of watch_paths.""" @abstractmethod - async def on_added(self, path: Path | list[Path]): - """Handle file added event.""" + async def on_added(self, path: str | list[str]): + """Handle file added event (relative paths).""" @abstractmethod - async def on_modified(self, path: Path | list[Path]): - """Handle file modified event.""" + async def on_modified(self, path: str | list[str]): + """Handle file modified event (relative paths).""" @abstractmethod - async def on_deleted(self, path: Path | list[Path]): - """Handle file deleted event.""" + async def on_deleted(self, path: str | list[str]): + """Handle file deleted event (relative paths).""" diff --git a/reme4/components/file_watcher/lite_file_watcher.py b/reme4/components/file_watcher/lite_file_watcher.py index df5370e6..681ec0ce 100644 --- a/reme4/components/file_watcher/lite_file_watcher.py +++ b/reme4/components/file_watcher/lite_file_watcher.py @@ -1,7 +1,6 @@ """Polling-based file watcher using watchfiles.""" import asyncio -from pathlib import Path from watchfiles import Change, awatch @@ -58,70 +57,68 @@ class LiteFileWatcher(BaseFileWatcher): async def _dispatch_changes(self, changes: set[tuple[Change, str]]): """Classify raw changes and dispatch to event handlers.""" - added = [Path(p) for c, p in changes if c == Change.added] - modified = [Path(p) for c, p in changes if c == Change.modified] - deleted = [Path(p) for c, p in changes if c == Change.deleted] - if added: - self.logger.info(f"Detected {len(added)} added file(s)") - await self.on_added(added) - if modified: - self.logger.info(f"Detected {len(modified)} modified file(s)") - await self.on_modified(modified) - if deleted: - self.logger.info(f"Detected {len(deleted)} deleted file(s)") - await self.on_deleted(deleted) + buckets: dict[Change, list[str]] = {Change.added: [], Change.modified: [], Change.deleted: []} + for c, p in changes: + if c in buckets: + buckets[c].append(self._get_relative_path(p)) + for change, handler, label in ( + (Change.added, self.on_added, "added"), + (Change.modified, self.on_modified, "modified"), + (Change.deleted, self.on_deleted, "deleted"), + ): + if buckets[change]: + self.logger.info(f"Detected {len(buckets[change])} {label} file(s)") + await handler(buckets[change]) async def update_store(self): if self.file_store is None: raise ValueError("file_store is not initialized!") - # Diff existing files against indexed entries by path and mtime. - existing = {str(p): p.stat().st_mtime for p in await self.scan_existing_files()} - indexed = {p: n.st_mtime for p, n in self.file_store.file_nodes.items()} - existing_keys, indexed_keys = set(existing), set(indexed) + existing: dict[str, float] = { + rel: abs_p.stat().st_mtime for rel, abs_p in (await self.scan_existing_files()).items() + } + indexed: dict[str, float] = {n.path: n.st_mtime for n in await self.file_store.file_graph.get_nodes()} - to_delete = indexed_keys - existing_keys - to_add = existing_keys - indexed_keys - to_modify = [p for p in existing_keys & indexed_keys if existing[p] != indexed[p]] + to_delete = list(indexed.keys() - existing.keys()) + to_add = list(existing.keys() - indexed.keys()) + to_modify = [p for p in existing.keys() & indexed.keys() if existing[p] != indexed[p]] if to_modify: self.logger.info(f"Updating {len(to_modify)} modified file(s)") - await self.on_modified([Path(p) for p in to_modify]) + await self.on_modified(to_modify) if to_delete: self.logger.info(f"Removing {len(to_delete)} deleted file(s)") - await self.on_deleted([Path(p) for p in to_delete]) + await self.on_deleted(to_delete) if to_add: self.logger.info(f"Indexing {len(to_add)} new file(s)") - await self.on_added([Path(p) for p in to_add]) + await self.on_added(to_add) if not to_modify and not to_delete and not to_add: self.logger.info("Store is up to date") - async def _parse_and_upsert(self, paths: list[Path], action: str): + async def _parse_and_upsert(self, paths: list[str], action: str): """Parse files and upsert into store. Shared by on_added / on_modified.""" if self.file_parser is None or self.file_store is None: raise RuntimeError("file_parser or file_store is not initialized!") parsed: list[tuple[FileNode, list[FileChunk]]] = [] - for p in paths: - if p.is_file(): - self.logger.info(f"{action} file: {p}") - parsed.append(await self.file_parser.parse(p)) + for rel in paths: + abs_path = self._get_absolute_path(rel) + if abs_path.is_file(): + self.logger.info(f"{action} file: {rel}") + parsed.append(await self.file_parser.parse(abs_path)) if parsed: - file_paths = [str(p) for p in paths if p.is_file()] - await self.file_store.delete_by_path(file_paths) + await self.file_store.delete_by_path([n.path for n, _ in parsed]) await self.file_store.upsert_file(parsed) - async def on_added(self, path: Path | list[Path]): - paths = [path] if isinstance(path, Path) else path - await self._parse_and_upsert(paths, "Adding") + async def on_added(self, path: str | list[str]): + await self._parse_and_upsert([path] if isinstance(path, str) else path, "Adding") - async def on_modified(self, path: Path | list[Path]): - paths = [path] if isinstance(path, Path) else path - await self._parse_and_upsert(paths, "Updating") + async def on_modified(self, path: str | list[str]): + await self._parse_and_upsert([path] if isinstance(path, str) else path, "Updating") - async def on_deleted(self, path: Path | list[Path]): + async def on_deleted(self, path: str | list[str]): if self.file_store is None: raise RuntimeError("file_store is not initialized!") - paths = [path] if isinstance(path, Path) else path + paths = [path] if isinstance(path, str) else path self.logger.info(f"Deleting {len(paths)} file(s)") - await self.file_store.delete_by_path([str(p) for p in paths]) + await self.file_store.delete_by_path(paths) diff --git a/reme4/components/keyword_index/bm25_index.py b/reme4/components/keyword_index/bm25_index.py index 863704d5..c64c70f3 100644 --- a/reme4/components/keyword_index/bm25_index.py +++ b/reme4/components/keyword_index/bm25_index.py @@ -10,6 +10,7 @@ from collections import Counter from typing import TypedDict from .base_keyword_index import BaseKeywordIndex +from ..component_registry import R class DocMeta(TypedDict): @@ -19,6 +20,7 @@ class DocMeta(TypedDict): token_ids: set[int] +@R.register("bm25") class BM25Index(BaseKeywordIndex): """BM25 search engine with file-based persistence. diff --git a/reme4/config/default.yaml b/reme4/config/default.yaml index 5eb447cf..e511c4bb 100644 --- a/reme4/config/default.yaml +++ b/reme4/config/default.yaml @@ -1,12 +1,69 @@ service: backend: http - host: 127.0.0.1 - port: 2333 +# backend: mcp jobs: - - name: demo - backend: base - description: "Echo back the incoming query (smoke test)." + - backend: base + name: demo_job + description: "demo job description" + parameters: + type: object + properties: + query: + type: string + description: "query" + min_score: + type: number + description: "min score" + default: 0.5 + required: + - query steps: - - backend: demo_echo + - backend: demo_echo_step1 + - backend: demo_echo_step2 +components: + # 1. tokenizer — 无依赖 + tokenizer: + default: + backend: regex + + # 2. embedding_model — 无依赖 + embedding_model: + default: + backend: openai + model_name: text-embedding-3-small + dimensions: 1536 + + # 3. file_graph — 无依赖 + file_graph: + default: + backend: local + + # 4. file_parser — 无依赖 + file_parser: + default: + backend: default + + # 5. keyword_index — 依赖 tokenizer + keyword_index: + default: + backend: bm25 + tokenizer: default + + # 6. file_store — 依赖 embedding_model / keyword_index / file_graph + file_store: + default: + backend: local + store_name: default + embedding_model: default + keyword_index: default + file_graph: default + + # 7. file_watcher — 依赖 file_store / file_parser + file_watcher: + default: + backend: lite + watch_paths: "." + file_store: default + file_parser: default \ No newline at end of file diff --git a/reme4/steps/__init__.py b/reme4/steps/__init__.py index 76f8c3a7..c8db6ea6 100644 --- a/reme4/steps/__init__.py +++ b/reme4/steps/__init__.py @@ -1,9 +1,10 @@ """steps""" -from . import demo from .base_step import BaseStep +from .demo import DemoEchoStep1, DemoEchoStep2 __all__ = [ "BaseStep", - "demo", + "DemoEchoStep1", + "DemoEchoStep2", ] diff --git a/reme4/steps/demo.py b/reme4/steps/demo.py new file mode 100644 index 00000000..9ad6e881 --- /dev/null +++ b/reme4/steps/demo.py @@ -0,0 +1,53 @@ +"""Demo steps for smoke-testing the application stack.""" + +from .base_step import BaseStep +from ..components import R + + +@R.register("demo_echo_step1") +class DemoEchoStep1(BaseStep): + """Read query/min_score from context, normalize, and write back for Step2.""" + + async def execute(self): + assert self.context is not None + query = self.context.get("query", "") + min_score = self.context.get("min_score", 0.5) + + self.logger.info(f"[{self.name}] query={query!r}, min_score={min_score}") + + processed_query = query.strip().lower() + adjusted_min_score = float(min_score) * 0.9 + + self.context["processed_query"] = processed_query + self.context["adjusted_min_score"] = adjusted_min_score + + return self.context.response + + +@R.register("demo_echo_step2") +class DemoEchoStep2(BaseStep): + """Consume Step1's outputs from context and emit the final response.""" + + async def execute(self): + assert self.context is not None + query = self.context.get("query", "") + min_score = self.context.get("min_score", 0.5) + processed_query = self.context.get("processed_query", "") + adjusted_min_score = self.context.get("adjusted_min_score", min_score) + + self.logger.info( + f"[{self.name}] query={query!r}, min_score={min_score}, " + f"processed_query={processed_query!r}, adjusted_min_score={adjusted_min_score}", + ) + + self.context.response.answer = f"echo: {processed_query} (min_score={adjusted_min_score})" + self.context.response.metadata.update( + { + "step": self.name, + "query": query, + "min_score": min_score, + "processed_query": processed_query, + "adjusted_min_score": adjusted_min_score, + }, + ) + return self.context.response diff --git a/reme4/steps/demo/__init__.py b/reme4/steps/demo/__init__.py deleted file mode 100644 index 057484a3..00000000 --- a/reme4/steps/demo/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Demo steps for smoke-testing the application stack.""" - -from ..base_step import BaseStep -from ...components.component_registry import R - - -@R.register("demo_echo") -class DemoEchoStep(BaseStep): - """Echo back the incoming `query` field into `response.answer`.""" - - async def execute(self): - assert self.context is not None - query = self.context.get("query", "") - self.context.response.answer = f"echo: {query}" - self.context.response.metadata["step"] = self.name - return self.context.response - - -__all__ = ["DemoEchoStep"]