This commit is contained in:
jinli.yl 2026-05-16 22:13:25 +08:00
parent 469f3bdee4
commit 3367d48ea1
12 changed files with 252 additions and 816 deletions

View file

@ -1,378 +0,0 @@
app_name: reme-expert
enable_logo: false
log_to_console: true
log_to_file: false
# Expert-exposure profile: every memory_* read + write primitive plus
# `sync` (event log) are surfaced as MCP tools. The agent picks whatever
# it needs — finest granularity, no opinion enforced beyond the
# wikilink-uniqueness gate.
#
# **No `ingest` tool.** Cold-path R-M-W in expert mode is owned by the
# host agent (Claude Code, with the `reme-expert` SKILL loaded), not by
# reme2's internal ReActAgent. The reme-expert plugin spawns a
# `reme-distiller` subagent at SessionEnd to handle distillation in its
# own context window, using the same memory_* primitives.
#
# Use when you want the agent to manage memory directly with full
# control. For an opinionated minimal surface (where reme2's internal
# Ingestor owns R-M-W), see ./service.yaml.
service:
backend: mcp
transport: stdio
sidecar_http: true
sidecar_http_host: "127.0.0.1"
sidecar_http_port: 8765
sidecar_info_path: "./vault/.reme/sidecar.json"
jobs:
# -- Write entry points ------------------------------------------------
- backend: base
name: sync
description: |
Hot-path event sync: idempotent upsert of an event FOLDER under
`events/{date}/{name}/`. The folder contains the index `{name}.md`
(Event schema, frontmatter + narrative + Materials footer) plus
any raw materials you pass — conversation snippets, tool outputs,
data dumps. The watcher indexes everything inside.
CONTINUITY MODEL: pick a stable `name` per logical thread and
call `sync` repeatedly through the task. Each call extends the
same folder:
* new `content` → appended under a `## Update — {iso}` section
* new `materials` → siblings (auto-suffix on filename collision)
* `topics` + `tags` merged (union) into frontmatter
* Materials footer regenerated to list every artifact in the folder
First call (folder doesn't exist) → CREATE; subsequent calls with
the same `name` while the event is `status: active` → APPEND.
If the event is `status: distilled` / `archived`, `sync` REFUSES
and returns a `suggested_name` so you start a fresh thread
instead of mutating prior cognition.
Zero LLM cost. Call CONTINUOUSLY through a task as facts land,
and especially at PreCompact to dump verbose raw text into
`materials` before context truncation loses it. The folder is
the unit `ingest` later reads from.
parameters:
type: object
required: [name]
properties:
name:
type: string
description: "kebab-case event identifier (folder + index stem). Reuse the same name across calls in one thread to keep extending the same folder."
description: { type: string, description: "one-line summary for index frontmatter (set on initial create only)" }
content: { type: string, description: "markdown body. Initial create: the body. Subsequent calls: appended as a `## Update — {iso}` section." }
topics:
type: array
items: { type: string }
description: "related topic wikilinks ('[[X]]' or '[[topics/X/X]]'). Unioned into frontmatter on append."
tags: { type: array, items: { type: string }, description: "free-form tags; unioned on append." }
materials:
type: array
description: "Raw artifacts written as siblings of the index inside the event folder. Filenames must be safe (letters/digits/dot/underscore/dash). On filename collision with an existing artifact, auto-suffixes (foo.txt → foo-2.txt)."
items:
type: object
required: [filename, content]
properties:
filename: { type: string, description: "e.g. 'raw-prompt.md', 'tool-output.txt', 'snapshot.json'" }
content: { type: string }
on_date: { type: string, description: "ISO date for events/{date}/ bucket; defaults to today" }
origin_session_id: { type: string, description: "set on initial create only" }
steps:
- backend: sync
# -- Read tools --------------------------------------------------------
- backend: base
name: memory_search
description: "Hybrid (vector + keyword) search over chunks."
parameters:
type: object
required: [query]
properties:
query: { type: string }
max_results: { type: integer, default: 5 }
min_score: { type: number, default: 0.1 }
paths: { type: array, items: { type: string } }
tags: { type: array, items: { type: string } }
exclude_paths: { type: array, items: { type: string } }
steps:
- backend: memory_search
- backend: base
name: memory_graph_search
description: |
Three-way fusion search: vector + keyword + graph (BFS over wikilinks).
Pulls in chunks reachable through linked topics/events that pure
relevance search would miss.
parameters:
type: object
properties:
query: { type: string }
max_results: { type: integer, default: 5 }
min_score: { type: number, default: 0.0 }
graph_depth: { type: integer, default: 1 }
seeds: { type: array, items: { type: string } }
paths: { type: array, items: { type: string } }
tags: { type: array, items: { type: string } }
exclude_paths: { type: array, items: { type: string } }
steps:
- backend: memory_graph_search
- backend: base
name: memory_get
description: "Read frontmatter + body of a single file."
parameters:
type: object
required: [path]
properties:
path: { type: string }
include_chunks: { type: boolean, default: false }
steps:
- backend: memory_get
- backend: base
name: memory_list
description: |
List indexed files filtered by frontmatter exact-match, tags, or
path prefix. Returns {items: [{path, metadata}], count}.
parameters:
type: object
properties:
metadata: { type: object }
tags: { type: array, items: { type: string } }
path_prefix: { type: string }
limit: { type: integer, default: 100 }
steps:
- backend: memory_list
- backend: base
name: memory_backlinks
description: "Files linking TO the given path (with edge predicates)."
parameters:
type: object
required: [path]
properties:
path: { type: string }
steps:
- backend: memory_backlinks
- backend: base
name: memory_links
description: "Files the given path links to (resolved, with edge predicates)."
parameters:
type: object
required: [path]
properties:
path: { type: string }
steps:
- backend: memory_links
- backend: base
name: memory_resolve_wikilink
description: |
Resolve a `[[target]]` wikilink to a vault path. Stem-form (`X`)
consults the file_store's stem index; path-form (`a/b` or `a/b.md`)
is anchored at the vault root.
parameters:
type: object
required: [wikilink]
properties:
wikilink: { type: string }
steps:
- backend: memory_resolve_wikilink
- backend: base
name: memory_count_tokens
description: "Estimate token count for a file body or raw text."
parameters:
type: object
properties:
path: { type: string }
text: { type: string }
steps:
- backend: memory_count_tokens
- backend: base
name: memory_lint
description: |
Read-only projection of the Maintainer's lint findings. Walks the
indexed files (optionally restricted to a path prefix) and returns
structural issues — broken wikilinks, schema violations, stem
collisions. Never mutates. The agent decides what to do with each
finding using existing memory_* primitives (rename, update,
property_update, archive, delete).
parameters:
type: object
properties:
target_prefix:
type: string
description: "restrict scan to relpaths starting with this prefix (e.g. 'events/2026-05-09/'). Empty string scans the whole vault."
steps:
- backend: memory_lint
# -- Write primitives (raw building blocks) ----------------------------
- backend: base
name: memory_create
description: |
Create a new file (raw primitive — no LLM reasoning). Prefer
`ingest` when you want an LLM curator. Two gates run unless
`force=true`: (1) path template — must be `topics/{folder}/{name}.md`,
`events/{date}/{name}/...`, or `Archive/...`; (2) wikilink uniqueness
— `[[stem]]` must resolve to ≤1 path post-create.
parameters:
type: object
required: [path]
properties:
path: { type: string }
metadata: { type: object }
content: { type: string }
overwrite: { type: boolean, default: false }
force:
type: boolean
default: false
description: "bypass BOTH the path-template gate and the wikilink-uniqueness gate. Use only when you intentionally need a non-template path or accept the ambiguity."
steps:
- backend: memory_create
- backend: base
name: memory_update
description: |
Edit-style content update: replace `old_string` with `new_string`
in the file body. For frontmatter changes use `memory_property_update`.
parameters:
type: object
required: [path, old_string, new_string]
properties:
path: { type: string }
old_string: { type: string }
new_string: { type: string }
replace_all: { type: boolean, default: false }
steps:
- backend: memory_update
- backend: base
name: memory_property_update
description: |
Update a single YAML frontmatter key. value=null deletes the key.
When key='status', enforces the active → distilled → archived
single-direction state machine (skip / reverse refused). Pass
`force=true` to bypass.
parameters:
type: object
required: [path, key]
properties:
path: { type: string }
key: { type: string }
value: {}
force:
type: boolean
default: false
description: "bypass the status state machine (only meaningful when key='status')."
steps:
- backend: memory_property_update
- backend: base
name: memory_rename
description: |
Rename a file and rewrite all incoming `[[wikilink]]` references
across the vault. Refuses on destination conflict or stem ambiguity.
parameters:
type: object
required: [old_path, new_path]
properties:
old_path: { type: string }
new_path: { type: string }
steps:
- backend: memory_rename
- backend: base
name: memory_delete
description: "Delete a file."
parameters:
type: object
required: [path]
properties:
path: { type: string }
steps:
- backend: memory_delete
- backend: base
name: memory_archive
description: |
Archive a file: flip `status: archived` then move under
`<vault>/<archive_dir>/<original_relative_path>`.
parameters:
type: object
required: [path]
properties:
path: { type: string }
archive_dir: { type: string, default: "Archive" }
steps:
- backend: memory_archive
components:
# Ingestor LLM (opt-in). Without this the Ingestor degrades to a
# direct create from explicit `target_path`; edits/renames/deletes
# require the LLM. Uncomment + provide LLM_API_KEY to enable.
#
# as_llm:
# default:
# backend: openai
# model_name: ${LLM_MODEL_NAME:-gpt-4o-mini}
# api_key: ${LLM_API_KEY}
# client_kwargs:
# base_url: ${LLM_BASE_URL:-https://api.openai.com/v1}
# stream: false
#
# as_llm_formatter:
# default:
# backend: openai
as_token_counter:
default:
backend: estimated
# Embedding is opt-in: leave embedding_model="" on file_store to run
# keyword-only; uncomment + flip to "default" to enable hybrid search.
#
# embedding_model:
# default:
# backend: openai
# model_name: ${EMBEDDING_MODEL_NAME:-text-embedding-3-small}
# dimensions: 1536
# pass_dimensions: false
# enable_cache: true
# max_batch_size: 10
# max_cache_size: 2000
# max_input_length: 8192
file_parser:
md:
backend: md
default:
backend: text
file_store:
default:
backend: local
embedding_model: ""
store_name: "reme"
db_path: "./vault/.reme"
working_dir: "./vault"
file_watcher:
default:
backend: full
file_store: default
default_parser: md
recursive: true
# Retriever (`hybrid`) is a Step, not a pre-instantiated component —
# the `memory_search` / `memory_graph_search` shells build it on
# demand. To tune defaults, pass knobs (`vector_weight`, `graph_*`,
# …) on the step config under each job below.

View file

@ -48,4 +48,70 @@
2. file_parser
a. 抽象基类 parse: @jinli
. 输入是path相对路径
ⅱ. 输出是FileMetadata & list[FileChunks] & list[FileEdge]
b. default parser 兼容老方案 @jinli
. 带overlap的chunking策略 不输出FileEdge
c. markdown parser @sen
. 根据markdown ast做chunk不需要overlap
ⅱ. 增加一个索引的chunk chunk_type @锦鲤 file_chunk_type content/index
ⅲ. 增加link的正则解析predicate:: [[path#anchor]]
3. file_store @sen
a. 抽象存储:
. filenode = file + path + st_mtime + metadata + list[FileEdge]
ⅱ. graph=dict[str, filenode] 内存+json
ⅲ. list[FileChunk] 存db
b. 抽象基类
. graphfellow dict的操作 update/get/set
ⅱ. chunks dict[str, list[chunk]]
1. delete_chunks_by_path
2. update_chunks_by_path
3. list_chunks_by_path
4. vector_search/keyword_search
ⅲ. 手写一个bm25检索
ⅳ. 【核心】检索机制 vector bm25 graph 如何进行融合
4. file_watcher @jinli
a. 抽象基类
. on_start:
1. file_store 的start 在前加载graphfile_watcher在后递归扫描目录
a. 通过ms_time对比graphon_change 进行改动
ⅱ. on_change:
1. 更新/增加:
a. delete_chunks_by_path 更新数据库
b. upate_chunks_by_path 更新数据库
c. 更新graph
2. 删除
a. delete_chunks_by_path 更新数据库
MemorySchema
1. markdown文件结构 @sen
a. formatter
. title
ⅱ. desc
ⅲ. tags
ⅳ.
2. memory文件结构目录
a. MEMORY.md
b. msg/files -> daily/YYYYMMDD/YYYYMMDD.md + xxxx.md
. YYYYMMDD.md
1. xxx -> xxxx.md
2. xxx -> xxxd.md
ⅱ.
c. daily -> topic/topic_l1/topic_l1.md + xxx.md + topic_l2
d. proactive
steps:
1. 治理(算法+LLM
a. 节点关联P0现有的链接做补充挖掘新的LLM的link
. /Users/yuli/workspace/ReMe/reme2/component/edge_extractor/llm_edge_extractor.py
ⅱ. 移动到steps
b. 节点整合/节点拆分/节点归档
c. 健康度检查
2. retrieve 调用store的检索
3. 原子stepsreme edit
4. 组合steps总结
a. - freq (every_n_turn、compact) -> daily_summarizer
b. topic (/dream ) -> topic_summarizer(daily_xx -> topic_xx)
c. proactive -> proactive_summarizer(personal_xxx -> proactive_query - pre_query

View file

@ -1,86 +0,0 @@
ReMe重构
1. 根目录
a. ✅ vault_root → working_dir @sen
2. file_parser
a. 抽象基类 parse: @jinli
. 输入是path相对路径
ⅱ. 输出是FileMetadata & list[FileChunks] & list[FileEdge]
b. default parser 兼容老方案 @jinli
. 带overlap的chunking策略 不输出FileEdge
c. markdown parser @sen
. 根据markdown ast做chunk不需要overlap
ⅱ. 增加一个索引的chunk chunk_type @锦鲤 file_chunk_type content/index
ⅲ. 增加link的正则解析predicate:: [[path#anchor]]
3. file_store @sen
a. 抽象存储:
. filenode = file + path + st_mtime + metadata + list[FileEdge]
ⅱ. graph=dict[str, filenode] 内存+json
ⅲ. list[FileChunk] 存db
b. 抽象基类
. graphfellow dict的操作 update/get/set
ⅱ. chunks dict[str, list[chunk]]
1. delete_chunks_by_path
2. update_chunks_by_path
3. list_chunks_by_path
4. vector_search/keyword_search
ⅲ. 手写一个bm25检索
ⅳ. 【核心】检索机制 vector bm25 graph 如何进行融合
4. file_watcher @jinli
a. 抽象基类
. on_start:
1. file_store 的start 在前加载graphfile_watcher在后递归扫描目录
a. 通过ms_time对比graphon_change 进行改动
ⅱ. on_change:
1. 更新/增加:
a. delete_chunks_by_path 更新数据库
b. upate_chunks_by_path 更新数据库
c. 更新graph
2. 删除
a. delete_chunks_by_path 更新数据库
MemorySchema
1. markdown文件结构 @sen
a. formatter
. title
ⅱ. desc
ⅲ. tags
ⅳ.
2. memory文件结构目录
a. MEMORY.md
b. msg/files -> daily/YYYYMMDD/YYYYMMDD.md + xxxx.md
. YYYYMMDD.md
1. xxx -> xxxx.md
2. xxx -> xxxd.md
ⅱ.
c. daily -> topic/topic_l1/topic_l1.md + xxx.md + topic_l2
d. proactive
steps:
1. 治理(算法+LLM
a. 节点关联P0现有的链接做补充挖掘新的LLM的link
. /Users/yuli/workspace/ReMe/reme2/component/edge_extractor/llm_edge_extractor.py
ⅱ. 移动到steps
b. 节点整合/节点拆分/节点归档
c. 健康度检查
2. retrieve 调用store的检索
3. 原子stepsreme edit
4. 组合steps总结
a. - freq (every_n_turn、compact) -> daily_summarizer
b. topic (/dream ) -> topic_summarizer(daily_xx -> topic_xx)
c. proactive -> proactive_summarizer(personal_xxx -> proactive_query - pre_query
Qwenpaw

View file

@ -1,270 +0,0 @@
app_name: reme-service
enable_logo: false
log_to_console: true
log_to_file: false
# Service profile: service-aligned MCP surface — three tools, one per
# memory service:
# `retrieve` — graph-aware hybrid retrieval (Retriever projection)
# `remember` — single write entry point (Ingestor projection)
# * mode=log → zero-LLM event-folder upsert
# * mode=distill → LLM R-M-W into topic graph
# `maintain` — vault hygiene sweep (Maintainer projection)
# runs lint + decay; merge/split require an LLM and
# are off by default.
#
# The agent calls `remember(mode=log, name=…, content=…, materials=…)`
# continuously through the task — picking a stable `name` per logical
# thread so each call extends the same event folder rather than
# fragmenting. At task completion (or PreCompact / SessionEnd) it
# calls `remember(mode=distill, content=…, related_paths=…)` once to
# distill the active events into topic-level cognition. `maintain` is
# typically cron-driven, but the agent can invoke it explicitly when
# it suspects vault drift.
#
# For full direct control over every memory_* primitive (raw writes,
# fine-grained reads, separate sync/ingest tools, memory_lint) see
# ./expert.yaml.
service:
backend: mcp
transport: stdio
sidecar_http: true
sidecar_http_host: "127.0.0.1"
sidecar_http_port: 8765
sidecar_info_path: "./vault/.reme/sidecar.json"
jobs:
- backend: base
name: retrieve
description: |
Graph-aware hybrid retrieval: vector + keyword + 1-hop wikilink
BFS fusion. Use for "what do I know about X" / "did I work on Y" /
"what's connected to [[Z]]". Anchor mode: include `[[Target]]` in
the query to seed BFS at that file. Topic-rooted mode: pass `seeds`
explicitly. Returns chunks ranked by combined relevance + graph
proximity, each tagged with `graph_hop`.
parameters:
type: object
properties:
query: { type: string }
max_results: { type: integer, default: 5 }
min_score: { type: number, default: 0.0 }
graph_depth:
type: integer
default: 1
description: "BFS hops from seeds. 1 covers immediate neighbors."
seeds:
type: array
items: { type: string }
description: "Explicit seed paths (topic-rooted mode)."
paths: { type: array, items: { type: string } }
tags: { type: array, items: { type: string } }
exclude_paths: { type: array, items: { type: string } }
steps:
- backend: memory_graph_search
- backend: base
name: remember
description: |
Single write entry point — projects the Ingestor service. Two
modes via the `mode` parameter:
* `mode: log` (zero LLM, hot path) — idempotent upsert of an
event FOLDER under `events/{date}/{name}/`. The folder
contains the index `{name}.md` (Event schema, frontmatter +
narrative + Materials footer) plus any raw materials you
pass — conversation snippets, tool outputs, data dumps. The
watcher indexes everything inside.
CONTINUITY MODEL: pick a stable `name` per logical thread
and call `remember(mode=log, ...)` repeatedly through the
task. Each call extends the same folder:
- new `content` → appended under a `## Update — {iso}`
section
- new `materials` → siblings (auto-suffix on filename
collision)
- `topics` + `tags` merged (union) into frontmatter
- Materials footer regenerated to list every artifact
First call (folder doesn't exist) → CREATE; subsequent calls
with the same `name` while the event is `status: active` →
APPEND. If `status: distilled` / `archived`, REFUSES and
returns `suggested_name` so you start a fresh thread instead
of mutating prior cognition. Call CONTINUOUSLY through a task
as facts land, especially at PreCompact to dump verbose raw
text into `materials` before context truncation.
* `mode: distill` (LLM R-M-W loop, cold path) — DEFAULT. Run
on EXPLICIT HANDOFF only: task completion / SessionEnd / when
you decide the working set is ready. Not a per-turn tool.
Hand off the working set in two interchangeable forms:
- `content` — inline material to distill (hint, summary, or
raw text)
- `related_paths` — pointers (event folder indexes; the
Ingestor follows `## Materials` to read each artifact,
individual material files, candidate topics).
The Ingestor reads the working set + linked topics, decides
which existing topics to update / create, and flips each
distilled event's status to "distilled". Returns an audit
trail.
parameters:
type: object
required: [content]
properties:
mode:
type: string
enum: [log, distill]
default: distill
description: "log = zero-LLM event-folder upsert (requires `name`); distill = LLM R-M-W into topic graph (default)."
# mode=log params
name:
type: string
description: "(mode=log) kebab-case event identifier (folder + index stem). Reuse the same name across calls in one thread to keep extending the same folder."
description:
type: string
description: "(mode=log) one-line summary for index frontmatter (set on initial create only)."
topics:
type: array
items: { type: string }
description: "(mode=log) related topic wikilinks. Unioned into frontmatter on append."
tags:
type: array
items: { type: string }
description: "(mode=log) free-form tags; unioned on append."
materials:
type: array
description: "(mode=log) raw artifacts written as siblings of the index. Filenames must be safe (letters/digits/dot/underscore/dash). Filename collision auto-suffixes (foo.txt → foo-2.txt)."
items:
type: object
required: [filename, content]
properties:
filename: { type: string, description: "e.g. 'raw-prompt.md', 'tool-output.txt'" }
content: { type: string }
on_date:
type: string
description: "(mode=log) ISO date for events/{date}/ bucket; defaults to today."
origin_session_id:
type: string
description: "(mode=log) optional source session identifier (set on initial create only)."
# shared / mode=distill params
content:
type: string
description: "Required. mode=log: markdown body for the index (initial create) or appended `## Update — {iso}` section. mode=distill: inline material the Ingestor distills — hint, summary, or raw text."
hint:
type: string
description: "(mode=distill) caller guidance about target / intent."
target_path:
type: string
description: "(mode=distill) optional suggested path; required for the no-LLM degraded path."
metadata:
type: object
description: "(mode=distill) suggested frontmatter for any new topic."
related_paths:
type: array
items: { type: string }
description: "(mode=distill) pointers — event folder indexes (Ingestor reads materials from `## Materials`), individual material files, or candidate topics."
steps:
- backend: ingestor
- backend: base
name: maintain
description: |
Vault hygiene sweep — projects the Maintainer service. One pass:
scan signals → propose ops → resolve conflicts → apply. Returns
an audit trail of what ran and what changed.
Default behavior runs `lint` (broken wikilinks, schema violations,
stem collisions — read-only diagnostics) and `decay` (move stale
events past their freshness window under `<vault>/Archive/`).
Merge / split require an LLM and are off unless explicitly opted
into via `ops`.
Typically cron-driven; the agent invokes it on demand when it
suspects vault drift (after a heavy session of edits, after a
bulk rename, etc.). Dry-run by default — flip `dry_run=false` to
apply changes.
parameters:
type: object
properties:
target_prefix:
type: string
description: "restrict scan to relpaths starting with this prefix (e.g. 'events/2026-05-09/'). Empty string scans the whole vault."
dry_run:
type: boolean
default: true
description: "if true (default) returns the plan without mutating; flip to false to actually apply lint+decay (and merge/split if opted in)."
ops:
type: array
items:
type: string
enum: [lint, decay, merge, split]
description: "subset of ops to run. Defaults to ['lint','decay']. Merge/split are LLM-driven and currently scaffolded — enabling them without an LLM is a no-op."
decay_days:
type: integer
description: "freshness window for the decay proposer (default = Maintainer constructor `decay_days`, typically 90)."
steps:
- backend: maintainer
components:
# Ingestor LLM (opt-in). Without this the Ingestor degrades to a
# direct create from explicit `target_path`; edits/renames/deletes
# require the LLM. Uncomment + provide LLM_API_KEY to enable.
#
# as_llm:
# default:
# backend: openai
# model_name: ${LLM_MODEL_NAME:-gpt-4o-mini}
# api_key: ${LLM_API_KEY}
# client_kwargs:
# base_url: ${LLM_BASE_URL:-https://api.openai.com/v1}
# stream: false
#
# as_llm_formatter:
# default:
# backend: openai
as_token_counter:
default:
backend: estimated
# Embedding is opt-in: leave embedding_model="" on file_store to run
# keyword-only; uncomment + flip to "default" to enable hybrid search.
#
# embedding_model:
# default:
# backend: openai
# model_name: ${EMBEDDING_MODEL_NAME:-text-embedding-3-small}
# dimensions: 1536
# pass_dimensions: false
# enable_cache: true
# max_batch_size: 10
# max_cache_size: 2000
# max_input_length: 8192
file_parser:
md:
backend: md
default:
backend: text
file_store:
default:
backend: local
embedding_model: ""
store_name: "reme"
db_path: "./vault/.reme"
working_dir: "./vault"
file_watcher:
default:
backend: full
file_store: default
default_parser: md
recursive: true
# Retriever (`hybrid`) is a Step, not a pre-instantiated component —
# the `query` shell builds it on demand. Tune defaults by attaching
# knobs to the `memory_graph_search` step under the `query` job above.

2
docs4/todo.md Normal file
View file

@ -0,0 +1,2 @@
1. 完善mcp_servers config
2. 完善mcp/http的服务测试

View file

@ -69,18 +69,29 @@ class BaseFileWatcher(BaseComponent):
return True
return any(path.endswith("." + s.strip(".")) for s in self.suffix_filters)
async def scan_existing_files(self) -> list[Path]:
"""Collect all watchable files under watch_paths."""
files: list[Path] = []
def _get_relative_path(self, path: str | Path) -> str:
"""Return path relative to working_dir, or absolute path if outside."""
file_path = Path(path).absolute()
try:
return str(file_path.relative_to(self.working_path.absolute()))
except ValueError:
return str(file_path)
def _get_absolute_path(self, path: str | Path) -> Path:
"""Return absolute path; relative paths are resolved against working_dir."""
p = Path(path)
return p if p.is_absolute() else self.working_path / p
async def scan_existing_files(self) -> dict[str, Path]:
"""Collect watchable files under watch_paths as {relative_path: absolute_path}."""
files: dict[str, Path] = {}
for path in self.watch_paths:
if not path.exists():
continue
if path.is_file():
if self.watch_filter(Change.added, str(path)):
files.append(path)
else:
items = path.rglob("*") if self.recursive else path.iterdir()
files.extend(p for p in items if p.is_file() and self.watch_filter(Change.added, str(p)))
candidates = [path] if path.is_file() else (path.rglob("*") if self.recursive else path.iterdir())
for p in candidates:
if p.is_file() and self.watch_filter(Change.added, str(p)):
files[self._get_relative_path(p)] = p.absolute()
return files
async def clear_store(self):
@ -94,7 +105,7 @@ class BaseFileWatcher(BaseComponent):
if self.file_store is None:
raise ValueError("file_store is not initialized!")
await self.file_store.clear()
await self.on_added(await self.scan_existing_files())
await self.on_added(list((await self.scan_existing_files()).keys()))
@abstractmethod
async def watch_loop(self):
@ -105,13 +116,13 @@ class BaseFileWatcher(BaseComponent):
"""Sync the store with the current state of watch_paths."""
@abstractmethod
async def on_added(self, path: Path | list[Path]):
"""Handle file added event."""
async def on_added(self, path: str | list[str]):
"""Handle file added event (relative paths)."""
@abstractmethod
async def on_modified(self, path: Path | list[Path]):
"""Handle file modified event."""
async def on_modified(self, path: str | list[str]):
"""Handle file modified event (relative paths)."""
@abstractmethod
async def on_deleted(self, path: Path | list[Path]):
"""Handle file deleted event."""
async def on_deleted(self, path: str | list[str]):
"""Handle file deleted event (relative paths)."""

View file

@ -1,7 +1,6 @@
"""Polling-based file watcher using watchfiles."""
import asyncio
from pathlib import Path
from watchfiles import Change, awatch
@ -58,70 +57,68 @@ class LiteFileWatcher(BaseFileWatcher):
async def _dispatch_changes(self, changes: set[tuple[Change, str]]):
"""Classify raw changes and dispatch to event handlers."""
added = [Path(p) for c, p in changes if c == Change.added]
modified = [Path(p) for c, p in changes if c == Change.modified]
deleted = [Path(p) for c, p in changes if c == Change.deleted]
if added:
self.logger.info(f"Detected {len(added)} added file(s)")
await self.on_added(added)
if modified:
self.logger.info(f"Detected {len(modified)} modified file(s)")
await self.on_modified(modified)
if deleted:
self.logger.info(f"Detected {len(deleted)} deleted file(s)")
await self.on_deleted(deleted)
buckets: dict[Change, list[str]] = {Change.added: [], Change.modified: [], Change.deleted: []}
for c, p in changes:
if c in buckets:
buckets[c].append(self._get_relative_path(p))
for change, handler, label in (
(Change.added, self.on_added, "added"),
(Change.modified, self.on_modified, "modified"),
(Change.deleted, self.on_deleted, "deleted"),
):
if buckets[change]:
self.logger.info(f"Detected {len(buckets[change])} {label} file(s)")
await handler(buckets[change])
async def update_store(self):
if self.file_store is None:
raise ValueError("file_store is not initialized!")
# Diff existing files against indexed entries by path and mtime.
existing = {str(p): p.stat().st_mtime for p in await self.scan_existing_files()}
indexed = {p: n.st_mtime for p, n in self.file_store.file_nodes.items()}
existing_keys, indexed_keys = set(existing), set(indexed)
existing: dict[str, float] = {
rel: abs_p.stat().st_mtime for rel, abs_p in (await self.scan_existing_files()).items()
}
indexed: dict[str, float] = {n.path: n.st_mtime for n in await self.file_store.file_graph.get_nodes()}
to_delete = indexed_keys - existing_keys
to_add = existing_keys - indexed_keys
to_modify = [p for p in existing_keys & indexed_keys if existing[p] != indexed[p]]
to_delete = list(indexed.keys() - existing.keys())
to_add = list(existing.keys() - indexed.keys())
to_modify = [p for p in existing.keys() & indexed.keys() if existing[p] != indexed[p]]
if to_modify:
self.logger.info(f"Updating {len(to_modify)} modified file(s)")
await self.on_modified([Path(p) for p in to_modify])
await self.on_modified(to_modify)
if to_delete:
self.logger.info(f"Removing {len(to_delete)} deleted file(s)")
await self.on_deleted([Path(p) for p in to_delete])
await self.on_deleted(to_delete)
if to_add:
self.logger.info(f"Indexing {len(to_add)} new file(s)")
await self.on_added([Path(p) for p in to_add])
await self.on_added(to_add)
if not to_modify and not to_delete and not to_add:
self.logger.info("Store is up to date")
async def _parse_and_upsert(self, paths: list[Path], action: str):
async def _parse_and_upsert(self, paths: list[str], action: str):
"""Parse files and upsert into store. Shared by on_added / on_modified."""
if self.file_parser is None or self.file_store is None:
raise RuntimeError("file_parser or file_store is not initialized!")
parsed: list[tuple[FileNode, list[FileChunk]]] = []
for p in paths:
if p.is_file():
self.logger.info(f"{action} file: {p}")
parsed.append(await self.file_parser.parse(p))
for rel in paths:
abs_path = self._get_absolute_path(rel)
if abs_path.is_file():
self.logger.info(f"{action} file: {rel}")
parsed.append(await self.file_parser.parse(abs_path))
if parsed:
file_paths = [str(p) for p in paths if p.is_file()]
await self.file_store.delete_by_path(file_paths)
await self.file_store.delete_by_path([n.path for n, _ in parsed])
await self.file_store.upsert_file(parsed)
async def on_added(self, path: Path | list[Path]):
paths = [path] if isinstance(path, Path) else path
await self._parse_and_upsert(paths, "Adding")
async def on_added(self, path: str | list[str]):
await self._parse_and_upsert([path] if isinstance(path, str) else path, "Adding")
async def on_modified(self, path: Path | list[Path]):
paths = [path] if isinstance(path, Path) else path
await self._parse_and_upsert(paths, "Updating")
async def on_modified(self, path: str | list[str]):
await self._parse_and_upsert([path] if isinstance(path, str) else path, "Updating")
async def on_deleted(self, path: Path | list[Path]):
async def on_deleted(self, path: str | list[str]):
if self.file_store is None:
raise RuntimeError("file_store is not initialized!")
paths = [path] if isinstance(path, Path) else path
paths = [path] if isinstance(path, str) else path
self.logger.info(f"Deleting {len(paths)} file(s)")
await self.file_store.delete_by_path([str(p) for p in paths])
await self.file_store.delete_by_path(paths)

View file

@ -10,6 +10,7 @@ from collections import Counter
from typing import TypedDict
from .base_keyword_index import BaseKeywordIndex
from ..component_registry import R
class DocMeta(TypedDict):
@ -19,6 +20,7 @@ class DocMeta(TypedDict):
token_ids: set[int]
@R.register("bm25")
class BM25Index(BaseKeywordIndex):
"""BM25 search engine with file-based persistence.

View file

@ -1,12 +1,69 @@
service:
backend: http
host: 127.0.0.1
port: 2333
# backend: mcp
jobs:
- name: demo
backend: base
description: "Echo back the incoming query (smoke test)."
- backend: base
name: demo_job
description: "demo job description"
parameters:
type: object
properties:
query:
type: string
description: "query"
min_score:
type: number
description: "min score"
default: 0.5
required:
- query
steps:
- backend: demo_echo
- backend: demo_echo_step1
- backend: demo_echo_step2
components:
# 1. tokenizer — 无依赖
tokenizer:
default:
backend: regex
# 2. embedding_model — 无依赖
embedding_model:
default:
backend: openai
model_name: text-embedding-3-small
dimensions: 1536
# 3. file_graph — 无依赖
file_graph:
default:
backend: local
# 4. file_parser — 无依赖
file_parser:
default:
backend: default
# 5. keyword_index — 依赖 tokenizer
keyword_index:
default:
backend: bm25
tokenizer: default
# 6. file_store — 依赖 embedding_model / keyword_index / file_graph
file_store:
default:
backend: local
store_name: default
embedding_model: default
keyword_index: default
file_graph: default
# 7. file_watcher — 依赖 file_store / file_parser
file_watcher:
default:
backend: lite
watch_paths: "."
file_store: default
file_parser: default

View file

@ -1,9 +1,10 @@
"""steps"""
from . import demo
from .base_step import BaseStep
from .demo import DemoEchoStep1, DemoEchoStep2
__all__ = [
"BaseStep",
"demo",
"DemoEchoStep1",
"DemoEchoStep2",
]

53
reme4/steps/demo.py Normal file
View file

@ -0,0 +1,53 @@
"""Demo steps for smoke-testing the application stack."""
from .base_step import BaseStep
from ..components import R
@R.register("demo_echo_step1")
class DemoEchoStep1(BaseStep):
"""Read query/min_score from context, normalize, and write back for Step2."""
async def execute(self):
assert self.context is not None
query = self.context.get("query", "")
min_score = self.context.get("min_score", 0.5)
self.logger.info(f"[{self.name}] query={query!r}, min_score={min_score}")
processed_query = query.strip().lower()
adjusted_min_score = float(min_score) * 0.9
self.context["processed_query"] = processed_query
self.context["adjusted_min_score"] = adjusted_min_score
return self.context.response
@R.register("demo_echo_step2")
class DemoEchoStep2(BaseStep):
"""Consume Step1's outputs from context and emit the final response."""
async def execute(self):
assert self.context is not None
query = self.context.get("query", "")
min_score = self.context.get("min_score", 0.5)
processed_query = self.context.get("processed_query", "")
adjusted_min_score = self.context.get("adjusted_min_score", min_score)
self.logger.info(
f"[{self.name}] query={query!r}, min_score={min_score}, "
f"processed_query={processed_query!r}, adjusted_min_score={adjusted_min_score}",
)
self.context.response.answer = f"echo: {processed_query} (min_score={adjusted_min_score})"
self.context.response.metadata.update(
{
"step": self.name,
"query": query,
"min_score": min_score,
"processed_query": processed_query,
"adjusted_min_score": adjusted_min_score,
},
)
return self.context.response

View file

@ -1,19 +0,0 @@
"""Demo steps for smoke-testing the application stack."""
from ..base_step import BaseStep
from ...components.component_registry import R
@R.register("demo_echo")
class DemoEchoStep(BaseStep):
"""Echo back the incoming `query` field into `response.answer`."""
async def execute(self):
assert self.context is not None
query = self.context.get("query", "")
self.context.response.answer = f"echo: {query}"
self.context.response.metadata["step"] = self.name
return self.context.response
__all__ = ["DemoEchoStep"]