From a2d76cc03420573b2fd1ebd8acd487078c6bb85f Mon Sep 17 00:00:00 2001 From: Sen Huang <48879559+ployts@users.noreply.github.com> Date: Thu, 4 Jun 2026 19:56:50 +0800 Subject: [PATCH] refactor(auto_dream): improve recall workflow and documentation (#272) * feat(file_store): add concurrency protection to LocalFileStore.dump() * feat(file_catalog): replace file_store with file_catalog in DreamStep * feat: add ChannelSink for Claude Code channel notifications * feat(auto-memory): add transcript_path support and enhance metadata --- docs4/old/auto_cognition_design.md | 2 +- docs4/old/auto_consolidate_design.md | 29 +- docs4/old/auto_dream_design.md | 89 ++- docs4/old/structure.md | 32 +- reme4/components/application_context.py | 8 +- reme4/components/job/background_job.py | 6 +- reme4/components/service/mcp_service.py | 106 ++- reme4/config/default.yaml | 34 +- reme4/steps/__init__.py | 18 +- reme4/steps/base_step.py | 14 +- reme4/steps/evolve/auto_dream.py | 684 ++++-------------- reme4/steps/evolve/auto_memory.py | 55 +- reme4/steps/evolve/dream.py | 499 +++++++++++++ .../evolve/{auto_dream.yaml => dream.yaml} | 227 ++++-- reme4/steps/file_io/daily_list.py | 1 + reme4/steps/file_io/daily_reindex.py | 5 + reme4/steps/file_io/delete.py | 11 + reme4/steps/file_io/frontmatter_delete.py | 4 + reme4/steps/file_io/frontmatter_read.py | 4 + reme4/steps/file_io/frontmatter_update.py | 2 + reme4/steps/file_io/move.py | 7 + reme4/steps/file_io/stat.py | 5 + reme4/steps/index/channel_notify.py | 67 ++ reme4/steps/index/claim_channel.py | 56 ++ reme4/steps/index/node_search.py | 158 ++++ reme4/steps/index/scan_changes.py | 141 +++- reme4/steps/index/traverse.py | 20 +- reme4/steps/index/watch_changes.py | 35 +- reme4/steps/transfer/download.py | 5 + reme4/steps/transfer/ingest.py | 6 + reme4/steps/transfer/upload.py | 6 + reme4/utils/service_utils.py | 1 + reme4/utils/transcript.py | 195 +++++ tests4/integration/test_dreamer_cli.sh | 68 -- tests4/integration/test_dreamer_inproc.py | 67 +- tests4/unit/test_auto_dream.py | 360 +++++++++ tests4/unit/test_background_steps.py | 18 +- tests4/unit/test_channel_notify.py | 104 +++ tests4/unit/test_channel_sink.py | 100 +++ 39 files changed, 2459 insertions(+), 790 deletions(-) create mode 100644 reme4/steps/evolve/dream.py rename reme4/steps/evolve/{auto_dream.yaml => dream.yaml} (77%) create mode 100644 reme4/steps/index/channel_notify.py create mode 100644 reme4/steps/index/claim_channel.py create mode 100644 reme4/steps/index/node_search.py create mode 100644 reme4/utils/transcript.py delete mode 100755 tests4/integration/test_dreamer_cli.sh create mode 100644 tests4/unit/test_auto_dream.py create mode 100644 tests4/unit/test_channel_notify.py create mode 100644 tests4/unit/test_channel_sink.py diff --git a/docs4/old/auto_cognition_design.md b/docs4/old/auto_cognition_design.md index 5daa3eb6..185631a7 100644 --- a/docs4/old/auto_cognition_design.md +++ b/docs4/old/auto_cognition_design.md @@ -318,7 +318,7 @@ auto-memory 写入的 daily event 节点也是图的一部分(承载 daily → d ## 11. 演进 / 待补 **当前实现状态**: -- ✅ Stage 1 dream 已实现并跑通(`reme4/steps/evolve/auto_dream.py` + `auto_dream.yaml`) +- ✅ Stage 1 dream 已实现并跑通(`reme4/steps/evolve/dream.py` + `dream.yaml`) - ⏳ Stage 2 consolidate split 部分将实现;dups / community / decay / archived 待实现 - ⏳ Stage 3 recall 增强未实现(当前 search.py 已有 vector + keyword + RRF + 一跳 expand) diff --git a/docs4/old/auto_consolidate_design.md b/docs4/old/auto_consolidate_design.md index e9d0d170..8061c75d 100644 --- a/docs4/old/auto_consolidate_design.md +++ b/docs4/old/auto_consolidate_design.md @@ -29,18 +29,20 @@ dream 写入是单点视角,有三类视野局限:**写入瞬间没有跨节点 | 4 | **腐败** | 长期不激活的痕迹 | 旧节点过时 / 半年没人读 / 内容已被矛盾 | archive | | 5 | **抽象缺位** | 跨多 instance 缺 schema | vault 只有原子节点,没有"主题层"视角承接全局问 | abstract | -### 0.1 五大动作 + 优先级 +### 0.1 四大动作 + 优先级 | 优先级 | 动作 | 解决问题 | 触发节奏 | 改 vault | 风险 | 收益 | |---|---|---|---|---|---|---| -| **P0** | **community detection** | (基础设施) | weekly batch | 否 | 0(只产 meta) | 基础(其它三个都靠它)| +| **P0** | **community detection** | (基础设施) | weekly batch | 否 | 0(只产 meta) | 基础(下游动作的依据)| | **P1** | **abstract** | 抽象缺位 | weekly batch(基于 P0) | 是(新建 summary) | 低(additive) | **最高**(GraphRAG 核心) | | **P2** | **merge** | 冗余 | weekly batch(基于 P0) | 是(合并 + retarget) | 高(lossy) | 中(消除可见冗余) | -| **P3** | **reinforce** | 稀疏 | weekly batch(基于 P0) | 是(additive 加 wikilink) | 低 | 低(retrieve multi-hop 已部分弥补)| | **(独立)** | **split** | 过载 | inline 写后(D3) | 是(拆 parent + children) | 低 | 中 | | **(独立)** | **archive** | 腐败 | daily batch | 软(meta 标记) | 0 | 中 | +| ~~P3 reinforce~~ | **已并入 dream synapse** | 稀疏 wikilink | 由 dream Phase 2 step 4 织突触承担 | (不在 consolidate 范围内) | — | — | -**关键论断**:**P1 比 P2 优先** —— abstract additive 失败可逆且回报最大;merge lossy 失败要回滚 inbound,价值是消除冗余(必要但不增能力)。 +**关键论断**: +- **P1 比 P2 优先** —— abstract additive 失败可逆且回报最大;merge lossy 失败要回滚 inbound,价值是消除冗余(必要但不增能力)。 +- **reinforce 已取消**(2026-06-02)—— 详 §4 标作废说明;wikilink 稀疏的解决方案是 dream Phase 2 在写入瞬间多召回 + 织突触(详 `auto_dream_design.md` §4.2.2),不再由 consolidate 周期补救。 ### 0.2 实施路径 @@ -51,9 +53,10 @@ M0: P0 community detection (基础设施) M1.1: P1 abstract (additive,最低风险开始改 vault) M1.2: P2 merge (lossy,高门槛 + 多数票) -M1.3: P3 reinforce (additive,价值最低,可缓做) -M2+: 多层 abstract (L2 super-community) / delete / typed predicate reinforce +M2+: 多层 abstract (L2 super-community) / delete + +reinforce: 不再排期 —— 已由 dream Phase 2 synapse 织突触承担 ``` ### 0.3 显式排除 @@ -365,7 +368,19 @@ audit 记录 + cooldown 设置 (winner 进 cooldown 2 weeks) --- -## 4. reinforce(P3,关系强化:补 dream 漏的 wikilink) +## 4. ~~reinforce~~(**已作废,2026-06-02**) + +> ⚠️ **本节作废,reinforce 已并入 dream Phase 2 synapse 织突触**(详 `auto_dream_design.md` §4.2.2)。理由: +> - reinforce 的本质 = "找语义相关但 wikilink 缺失的节点对,补 wikilink" +> - 但 dream Phase 2 在写入新节点瞬间已经在做同样的事(多召回 + 内化判 related + 织 `[[Y.md]]`) +> - 让 consolidate 周期事后补 wikilink = dream RECALL 不充分的兜底,与其兜底不如把 dream 召回做强 +> - F-2 自然守住:dream 只动新节点 body(自己的 subject),不需要 consolidate 改 leaf body 这种 F-2 破例 +> +> **新立场**:wikilink 的稀疏由 dream Phase 2 在写入瞬间一次性解决,vault 不维护"事后周期补 wikilink"的通道(`auto_cognition_design.md` §9.2 立场:关系建立在写入瞬间)。详 `hierarchical_summary.md` §13.2 Q4。 +> +> 以下保留原 reinforce 设计内容作为历史快照,**不实施**。 + +**(以下内容已作废,仅作历史快照)** **类比**:NREM 突触强化 LTP —— 反复共激活的连接被强化。 diff --git a/docs4/old/auto_dream_design.md b/docs4/old/auto_dream_design.md index b9924fac..83b8b8d7 100644 --- a/docs4/old/auto_dream_design.md +++ b/docs4/old/auto_dream_design.md @@ -34,7 +34,7 @@ dream 设计回答四个问题:**桶**怎么布局 / **节点**长什么样 / ** |---|---| | **物理几何** | `digest//.md`;**浅桶一层**(顶多两层),桶内 flat | | **bucket 角色** | **仅承担物理归档 + OS-level 浏览锚点**;不承担语义本体角色 —— 主题由图中节点表达 | -| **bucket 集合** | **代码内 hard-coded**(`reme4/steps/evolve/auto_dream.py` 的 `BUCKETS` 常量),不通过配置外置,不由 dreamer / maintainer 动态生成 —— 三桶设定是 dream 模型本身的一部分(Phase 2 prompt 按 bucket 专化),不是可调参数 | +| **bucket 集合** | **代码内 hard-coded**(`reme4/steps/evolve/dream.py` 的 `BUCKETS` 常量),不通过配置外置,不由 dreamer / maintainer 动态生成 —— 三桶设定是 dream 模型本身的一部分(Phase 2 prompt 按 bucket 专化),不是可调参数 | | **集合视图** | 桶名内嵌在 prompt 中(extract 阶段三桶判别启发 + 三份独立 integrate prompt);不再生成独立 `_buckets.md` 视图 | | **初始化** | opinionated **三桶**,按"答什么问 + 谁在问"划分:`procedure`(答"怎么做 X" —— 步骤 / 方法 / runbook)/ `personal`(答"X 是谁 / 喜欢什么 / 不要做什么" —— 用户 / 团队 specific 身份 + 偏好)/ `wiki`(答"X 是什么 / 发生了什么 / 决策依据是什么" —— 通用知识 / 定义 / 原则 / 观察 / 决策先例;**也是默认兜底**) | | **bucket 主页** | 不强制存在;split 累积出层级时 parent 节点天然成为浏览主页(中心性涌现,非架构必需) | @@ -179,11 +179,83 @@ agent 上报 IntegrateOutcome {action, target_path} **Phase 2 的 bucket 专化**:三桶各有独立 system prompt,因为各桶的 body 形态、决策偏置不同 —— `procedure` 节点是 runbook 风(触发 / 步骤 / 前置 / 失败模式),`personal` 节点是规则风(rule + Why + How to apply),`wiki` 节点是百科风(定义 + 性质 + 关系)。共用一份通用 prompt 会让"应该写成什么样"的指导被稀释,bucket 信号靠一段 if-this-then-that 散文承载,效果劣于让每桶自带专属 prompt。 -#### 4.2.2 召回二段 +#### 4.2.2 召回 → 内化分类 → 决策 → 织突触(ReAct agent 一体完成) -**RECALL = search + traverse**:search 给关键词 + 向量 RRF 命中;只要 search 在 `digest/` 下返回任何 hit,就对 top hit 跑 `traverse depth=2 direction=both`。理由是 search 关键词导向,会漏掉用不同术语归档的语义相邻抽象,那些常常一跳之外。search 在 `digest/` 下完全无命中 → 无 traverse 起点 → 候选集为空 → 直接 CREATE。 +Phase 2 是单个 ReAct agent 在一个 loop 内完成 4 件事 —— **不拆 stage,不引入外部机械步骤**,只通过 prompt 引导 agent 把 dedup 与 synapse 这两类判断都做透。当前默认 `search(limit=5)` 不够,prompt 已显式引导更深召回。 -**HIT = frontmatter_read + read**:渐进披露 —— 先 `frontmatter_read` 读 `name + description` 廉价 triage 淘汰明显无关候选,剩下的再 `read` 整 body。**不可仅凭 chunk 片段或 frontmatter 决定 UPDATE**,body 才是判定依据。 +**4 步流程**(整段由 ReAct agent 自主组织调用): + +| # | 步 | 关键动作 | +|---|---|---| +| 1 | **召回 —— 多角度宽召** | 显式 `limit=20-30` × 两轮 search(一次 hybrid,一次 `vector_weight=1.0` 纯语义)+ `traverse depth=2` 拓扑补充 | +| 2 | **内化分类** | `frontmatter_read` triage + 必要时 `read` body;对每个候选**内化打 label**(只在思考中分类,不输出):`same_abstraction` / `related` / `unrelated` | +| 3 | **决策** | 0 个 `same_abstraction` → CREATE;1 个 → UPDATE(选 flavor) | +| 4 | **织突触** | CREATE 或 UPDATE 都把所有 `related` 候选织入 body 作 `[[Y.md]]`;CREATE 一次性织全;UPDATE additive 加 wikilink | + +**两类内化判断的本质**: + +| 判断 | 服务 | 输出形态 | +|---|---|---| +| **同抽象?**(dedup)| 决定 CREATE / UPDATE | 0/1 个 target(决策面排他) | +| **相关?**(synapse)| 决定织哪些 wikilink | N 个 related 候选(决策面累加) | + +两者是同一个 ReAct agent 在看完 candidates 后的**两层独立判断**,共享同一批召回结果,**不需要分两轮 LLM 调用**。 + +**召回**(对应 prompt step 1):dream 用专属的 `node_search`(`reme4/steps/index/node_search.py`),**不**用通用 `search`,**也不用 `traverse`** —— 详 §4.2.2.1(traverse 是 retrieve-time 子图挖掘工具,跟 dream 写入场景错位)。 + +| 调用 | 找什么 | +|---|---| +| `node_search(query=<...>, limit=20-30)` | digest 内节点级 hybrid 召回(vector + BM25 RRF),返回 path + frontmatter | + +**召回结果服务两类判断**:dedup(`same_abstraction` label,是否同抽象 → CREATE / UPDATE)和 synapse(`related` label,是否相关 → 织 wikilink)是 LLM 在**同一批 candidates** 上的两类内化 label。原"两轮 search(hybrid + vector_only)"是设计冗余 —— 同一批候选 LLM 自己能判 same/related/unrelated,模式切换无意义。**调用次数由 agent 自决**:一次通常够;若 unit 跨多个概念维度,agent 可发起多次不同 query 的召回,prompt 不强约束。 + +**HIT = `node_search` 返回 + read**:`node_search` 已内嵌返回每个 hit 的 frontmatter(`name + description`),agent 直接据此 triage,**不需要额外调 `frontmatter_read` 批量取 metadata**;仅对需要看 body 的少数候选用 `read`。**不可仅凭 frontmatter 决定 UPDATE**,body 才是判定依据。 + +##### 4.2.2.1 node_search vs 通用 search 的差别 + 为什么 dream 不用 traverse + +**node_search vs 通用 search**:dream 的召回需求跟外部 agent 的 RAG 检索**结构性不同**,因此用专属 step 而非复用 `search`: + +| 维度 | 通用 `search`(外部 agent)| `node_search`(dream Phase 2) | +|---|---|---| +| 用户 | 用户/外部 agent 的自然语言 query | dream 内部生成的 unit.summary | +| 结果粒度 | **chunk 级**(可能同一 node 多个 chunk)| **node 级**(同 path 聚合 max score)| +| 返回信息 | 完整 chunk text + scores | **path + name + description**(frontmatter 内嵌,无 body)| +| 范围 | 全 vault(daily / resource / digest) | **digest-only**(dream 永远只在 digest 找候选)| +| expand_links | 默认 `True`(给 agent 更多上下文)| **永远 `False`**(synapse 找的就是未 link 的)| +| 默认 limit | 5 | **20**(dream 需要宽召覆盖 synapse)| + +复用通用 `search` 会让 dream 拿到的候选**既粒度不对**(chunk 级,同 node 多次出现)**又信息冗余**(chunk text 不必要)**又被噪声污染**(daily / resource hits 永远不是 dream 的 UPDATE 候选)**又召回偏窄**(expand_links 把已 link 的拖回来,挤掉真正未 link 的 synapse 候选)。所以 dream 需要自己的 `node_search`。 + +**为什么 dream toolkit 不包含 traverse(或 dream_traverse)** —— traverse 是 **retrieve-time 子图挖掘工具**,跟 dream 写入场景**结构性错位**: + +| 维度 | traverse 的本性(retrieve / RAG)| dream 的真实需求(写入)| +|---|---|---| +| 方向 | 从已知中心向外扩散 | 从外部新材料找 vault 内相关候选 | +| 输入 | 已知种子节点 | 新材料的 unit.summary | +| 输出语义 | "X 的子图"(给读者上下文) | "X 应该 link 到哪些 Y" | +| 图遍历的角色 | 主操作 | 召回兜底(可有可无) | + +dream 写新节点要回答"vault 中谁跟我相关",这是**召回**问题(给 query 找相关),不是**遍历**问题(给中心找邻居)。**召回工具 = node_search;遍历工具 = traverse(留给 retrieve / 外部 agent 用)。dream 不需要遍历**。 + +(早期曾实现 `dream_traverse` 准备作为 dream toolkit 一员,后撤销 —— 实测拓扑遍历 vs vector 召回重叠率 ~95%,真正独特贡献 < 2%,且引入 LLM 调用 / 上下文 / 复杂度成本。详 git log。) + +**node_search 参数极简**(`query / limit` 两个):**mode 不需要**(同一批候选服务双判断);**exclude_paths 不需要**(self 由 LLM 自己识别,frontmatter 内嵌让 agent 一眼看出"这就是我");**min_score 不需要**(RRF 分数范围 0~0.025,跟 cosine 0~1 量纲完全不同,召回深度由 `limit` 控制就够)。**调用次数 agent 自决**:prompt 不约束"必须一次",unit 跨多个概念维度时 agent 可多次召回。 + +**node_search 召回算法:weighted node-level RRF**(vector + BM25 hybrid): + +- vector + BM25 各自独立召回 → 各自得到 chunk list(按各自 score 排序) +- 同 path 多 chunk 合并:取该 path 在两个 list 中的 max chunk score 位置作为 node rank +- RRF 融合:`score(path) = vector_weight × 1/(60 + rank_v) + (1-vector_weight) × 1/(60 + rank_k)` +- `vector_weight=0.7`(默认),vector 主导,BM25 作为兜底(覆盖专有名词 / 缩写等 embedding 可能 struggle 的字面 case) +- 输出 score 是 RRF 分(0~0.025 量级,不是 cosine);LLM 不依赖具体分数,内化判 same/related/unrelated + +**reinforce 并入立场**(对照 `auto_consolidate_design.md` §4 标作废):reinforce 不再是独立的 consolidate 动作 —— 它就是 step 4 的"织突触"。新节点写入瞬间一次性建立关系,vault 不维护"事后周期 batch 补 wikilink"的通道。F-2 自然守住 —— dream 只动新节点 body,不动其它节点。 + +**关键约束**(诚实承认): +- **写入即定型** —— 今天没织的 wikilink 以后没机会再织;vault 单调演化 +- **一次性 commit,无事后兜底** —— prompt 明示"宁可多织"(false positive 一眼能否决;false negative 永远沉默) +- **召回深度取决于 prompt 引导 + agent 配合** —— 不引入外部机械召回 step;prompt 已明示 `limit=20-30 × 两轮`,但仍是 ReAct agent 的开放执行 +- **dedup 与 synapse 在一次 LLM 调用内完成** —— 不拆独立 stage,共享召回结果,内化分类是免费的 #### 4.2.3 UPDATE 三种 flavor @@ -207,6 +279,8 @@ agent 上报 IntegrateOutcome {action, target_path} - **0 出边节点合法**(没识别到合适邻居),后续 dream 进入时其它节点可以反向链回来 —— 不强求 LLM 一次性给全 - **dream 漏判去重**(同概念建成新节点)→ 不主动兜底,接受重复;若 vault 累积明显重复,由 auto-consolidate 的 dups 检测周期 batch 产报告(`auto_consolidate_design.md` §3) - **召回不做 bucket 粗筛** —— LLM 拥有完整跨桶视野,可识别"概念跨桶同抽象"(例如同一原则在 wiki 已有节点而 Phase 1 把新材料归入 personal,此时 UPDATE wiki 节点而非新建 personal 节点) +- **reinforce 已并入 dream synapse recall** —— 不存在独立的 reinforce 动作或周期 batch;突触构建(原 `auto_consolidate_design.md` §4 reinforce 的职责)在 dream Phase 2 synapse recall 阶段完成,新节点写入瞬间织全(详 §4.2.2) +- **vault 不维护事后补 wikilink 通道** —— 上一条的直接推论;cognition §9.2 立场("关系建立在写入瞬间")在此自然守住 **provenance 写出**: - 行文中自然带:"... 该模式最早出现在 [[daily/2026/05/15.md]] 的实践中" @@ -266,10 +340,13 @@ agent 上报 IntegrateOutcome {action, target_path} 本文档覆盖 dream 模型(桶 / 节点 / 边 / 演化)。组织端实现清单(M split / D 检测 / CAS 框架)见 `auto_consolidate_design.md` §10。 -- ✅ **dream step 实现** —— Phase 1 extract(识别抽象 + 分配 bucket)+ Phase 2 integrate(per sub-unit,**bucket-specific prompt 分发**;`reme4/steps/evolve/auto_dream.py` + `auto_dream.yaml`,与 `auto_memory` 同级同形) -- ✅ **三桶 hard-coded** —— `procedure / personal / wiki`,`BUCKETS` 常量在 `dreamer.py` 顶部,Phase 1 通过 `MemoryUnit.bucket: Literal[...]` 由 Pydantic 强制约束 +- ✅ **dream step 实现** —— Phase 1 extract(识别抽象 + 分配 bucket)+ Phase 2 integrate(per sub-unit,**bucket-specific prompt 分发**;`reme4/steps/evolve/dream.py` + `dream.yaml`,与 `auto_memory` 同级同形) +- ✅ **三桶 hard-coded** —— `procedure / personal / wiki`,`BUCKETS` 常量在 `dream.py` 顶部,Phase 1 通过 `MemoryUnit.bucket: Literal[...]` 由 Pydantic 强制约束 - ✅ **provenance prompt 规范** —— `derived_from:: [[daily/...]]` / `[[resource/...]]` 强制(三桶 prompt 各自重申) - ❌ ~~**边守恒校验工具**~~ —— 早期 `digest_edit` 子类的 outbound diff 校验已随子类一并移除(切到 canonical `edit`);E-1 现由 prompt 自律,详 §4.4 - ❌ ~~**bucket 集合配置外置**~~ —— 撤销:三桶是 dream 模型本身的一部分,不做配置参数(`vault.yaml` 不再承载 `digest.buckets`,`_buckets.md` 视图也不再生成) +- 🆕 **Phase 2 召回拆 dedup / synapse**(2026-06-02 沉淀,详 §4.2.2)—— 当前 prompt 共用一次 `search(limit=5)`,既不够 dedup 精度也不够 synapse 覆盖;落地:`dream.yaml` 6 处(en + zh × 3 buckets)Recall 段改写,加 synapse 模式说明 + 写入即定型纪律 +- 🆕 **`file_store.default.embedding_model` 启用**(blocker)—— `default.yaml` 当前 `""`,synapse recall 用 vector_weight=1.0 模式必须开启;否则 `search` 退化为纯 BM25,dedup 也劣化 +- 🆕 **reinforce 并入立场写入**(详 §4.2.2)—— 与 `auto_consolidate_design.md` §4 标作废同步;`hierarchical_summary.md` §13.2 Q4 标解决 实现进入 `reme4/steps/evolve/` 时,本文档与 `auto_memory_design.md` / `auto_consolidate_design.md` / `auto_cognition_design.md` 共同作为契约依据。 diff --git a/docs4/old/structure.md b/docs4/old/structure.md index 56bb89dc..8cdb0443 100644 --- a/docs4/old/structure.md +++ b/docs4/old/structure.md @@ -173,7 +173,7 @@ Service 与 Runtime 是同一份 vault 上的两个进程角色: | 不变量 | 内容 | 来源 | |---|---|---| -| **I-1** | agent 不直接写 digest(digest 写权只属 digester / maintainer) | §2.4 | +| **I-1** | agent 不直接写 digest(digest 写权只属 dreamer / maintainer) | §2.4 | | **I-2** | daily folder 单作者(同 folder 不并发改) | §2.4 | | **I-3** | resource 内容不可变,只允许 metadata appendable | §2.4 | | **I-4** | 三层共用同一套 wikilink 索引,跨层引用全靠 wikilink | §2.4 | @@ -215,10 +215,10 @@ Service 与 Runtime 是同一份 vault 上的两个进程角色: | 维度 | resource/ | daily/ | digest/ | |---|---|---|---| | **组织主轴** | 时间(`/`) | 时间 + 任务(`//`) | 语义(`//...`,任意嵌套) | -| **写权归属** | 入流通道唯一(webhook / upload / pull) | agent(写入任务过程) | digester / maintainer(无 agent 直写) | +| **写权归属** | 入流通道唯一(webhook / upload / pull) | agent(写入任务过程) | dreamer / maintainer(无 agent 直写) | | **可变性** | 不可变,只追加新文件 | folder 内可反复更新 | 单节点可演化,可被合并/拆分/移动 | | **不变量** | 写入即冻结,原文永不变 | folder 名 = summary note 名(可移动单元);同 slug 同日只一份 | slug 全局唯一;每 folder 有 canonical entry;wikilink 全路径 | -| **谁在用** | agent(查原文)、digester(双源输入之一) | agent(自己的工作记录)、digester(双源输入之一) | agent(召回主目标)、maintainer(自维护对象) | +| **谁在用** | agent(查原文)、dreamer(双源输入之一) | agent(自己的工作记录)、dreamer(双源输入之一) | agent(召回主目标)、maintainer(自维护对象) | ### 2.3 写入纪律:并行写入 + 双源合流 @@ -239,12 +239,12 @@ Service 与 Runtime 是同一份 vault 上的两个进程角色: ┌──────────────┐ ◄──╮ │ digest/ │ │ maintain │ 可重组 │ ────╯ (in-place, - │ (digester + │ fold-only) + │ (dreamer + │ fold-only) │ maintainer) │ └──────────────┘ ``` -写权按这个**两层并行 → 单层合流**的拓扑分配:resource 写权专属 ingester(外部入流通道),daily 写权专属 agent(sync 落入,可响应 notify 或自身任务驱动),digest 写权专属 digester + maintainer。**resource 与 daily 之间互不写入**(agent 不动 resource,ingester 不动 daily);任何一层都不能反向改写它的上游。这是整个架构的脊梁。 +写权按这个**两层并行 → 单层合流**的拓扑分配:resource 写权专属 ingester(外部入流通道),daily 写权专属 agent(sync 落入,可响应 notify 或自身任务驱动),digest 写权专属 dreamer + maintainer。**resource 与 daily 之间互不写入**(agent 不动 resource,ingester 不动 daily);任何一层都不能反向改写它的上游。这是整个架构的脊梁。 ### 2.4 不变量(永远成立) @@ -310,12 +310,12 @@ Service 与 Runtime 是同一份 vault 上的两个进程角色: | 维度 | 内容 | |---|---| -| 主体 | Reme Runtime(`digester` 模块,LLM-driven) | +| 主体 | Reme Runtime(`dreamer` 模块,LLM-driven) | | 输入 | 一组待蒸馏的 daily folder + 相关 resource(双源合流;通常以 daily 任务为线索,顺着 wikilink / 同主题搜索拉入相关 resource 原文) | | 产出 | digest 中 0~N 个新节点 或 已有节点的更新;新节点必须用 wikilink 反指至少一个上游来源 | | 不变量 | resource / daily 正文 0 修改;digest 新节点必须 wikilink 反指上游(provenance);digest 节点遵守第 2.4 节列的不变量 | | Provenance | digest → daily / resource 双源链条可达(资料源是 resource 时直接反指,任务过程是 daily 时反指 daily 进而可达 resource) | -| 反例 | digester 改写 resource;digester 改写 daily 正文 | +| 反例 | dreamer 改写 resource;dreamer 改写 daily 正文 | ### 3.6 maintain:digest → digest 折叠 @@ -605,12 +605,12 @@ L2 只关心"vault 当前是什么样",**无业务语义** —— 不知道 dail | **ingester** | ingest | External push / pull | × | 原样落 resource + 抽 frontmatter + 入索引 | | **notifier** | notify | Reme background(cron + L2 资源自治状态阈值) | × | 从 L2 资源自治状态选候选 → 写 L2 推送队列;Service MCP 拿走 | | **synchronizer** | sync | Agent on-demand | ✓ | 把当下事件织入 daily 工作叙事 | -| **digester** | digest | Reme background | ✓ | resource + daily 双源合流成 digest 长期条目 | +| **dreamer** | digest | Reme background | ✓ | resource + daily 双源合流成 digest 长期条目 | | **maintainer** | maintain | Reme background | ✓ | digest topic tree 的**密度折叠**(fold-only) | `retrieve` 不构成独立 L4 模块,理由见 §7.4。 -**三个 reme 自治模块**:notifier(机械)、digester(LLM)、maintainer(LLM)。三者都由 scheduler 触发,都消费 L2 自治状态,但只有 notifier 是机械的 —— 候选选择不需要 LLM,LLM 决策在 agent 侧的 sync。 +**三个 reme 自治模块**:notifier(机械)、dreamer(LLM)、maintainer(LLM)。三者都由 scheduler 触发,都消费 L2 自治状态,但只有 notifier 是机械的 —— 候选选择不需要 LLM,LLM 决策在 agent 侧的 sync。 ### 7.2 对称结构 @@ -621,7 +621,7 @@ L2 只关心"vault 当前是什么样",**无业务语义** —— 不知道 dail Inbound: ingester Attention: notifier Working: synchronizer - Sink: digester + Sink: dreamer Organization: maintainer (fold-only) ``` @@ -632,7 +632,7 @@ L2 只关心"vault 当前是什么样",**无业务语义** —— 不知道 dail | ingester | 外部异构格式 → vault 统一文件 | | notifier | L2 资源自治状态 → agent 注意力(`notify` 推送) | | synchronizer | agent 事件流 → 工作过程叙事(写 hot) | -| digester | 工作过程 + 原始资料 → 长期知识(双源合流,写 cold) | +| dreamer | 工作过程 + 原始资料 → 长期知识(双源合流,写 cold) | | maintainer | 散乱叶子 → 有层次的 topic tree(组织 cold) | ingester 和 notifier 是机械(确定性阈值/流水线);其它三个是 LLM 决策模块,各自跨越一层语义鸿沟。 @@ -728,11 +728,11 @@ Schema(资料的 frontmatter / wikilink / 章节约定)是横跨三层、各写 | # | 反例 | 违反的不变量 | |---|---|---| -| ✗-1 | Agent 通过任意 verb 直接写 digest | I-1(digest 写权只属 digester / maintainer) | +| ✗-1 | Agent 通过任意 verb 直接写 digest | I-1(digest 写权只属 dreamer / maintainer) | | ✗-2 | 多 agent 并发改同一个 daily folder | I-2(daily 单作者) | | ✗-3 | 任何动作改写 resource 的原文 | I-3(resource immutable) | | ✗-4 | 跨层引用引入第二套机制(hash-id / external ref / SQL) | I-4(wikilink 是唯一跨层载体) | -| ✗-5 | digester 改写 daily 正文 | `digest` 不变量(§3.5) | +| ✗-5 | dreamer 改写 daily 正文 | `digest` 不变量(§3.5) | | ✗-6 | maintain 改写 daily / resource 的语义内容 | `maintain` 不变量(§3.6) | | ✗-7 | `notify` 维护 resource 上的 `referenced_by` 反指 | `notify` 完全单向(§3.3) | | ✗-8 | 把 state / semantic / topological 合并成单一 read verb | R-1 | @@ -764,13 +764,13 @@ Schema(资料的 frontmatter / wikilink / 章节约定)是横跨三层、各写 | **L5 Service** | 服务 agent 请求的进程(HTTP / MCP);执行栈最上层;也是 notify 的 MCP transport | | **L5 Runtime** | 自治维护 vault 的进程;scheduler 在其中按 L2 自治状态阈值触发 background Action | | **L4 Action** | 6 类动作语义:ingest / notify / sync / retrieve / digest / maintain | -| **L4 模块** | 实现 Action 的架构角色;五个:ingester / notifier / synchronizer / digester / maintainer(retrieve 不构成独立模块) | +| **L4 模块** | 实现 Action 的架构角色;五个:ingester / notifier / synchronizer / dreamer / maintainer(retrieve 不构成独立模块) | | **ingester** | L4 模块,机械:外部源原样落 resource + 抽 frontmatter + 入索引 | | **notifier** | L4 模块,机械:从 L2 资源自治状态选 notify 候选 → 写 L2 推送队列;Service MCP 拿走推给 agent | | **synchronizer** | L4 模块,LLM-driven:agent 事件织入 daily 工作叙事;响应 notify 的也走这里 | -| **digester** | L4 模块,LLM-driven:resource + daily 双源合流为 digest 长期条目 | +| **dreamer** | L4 模块,LLM-driven:resource + daily 双源合流为 digest 长期条目 | | **maintainer** | L4 模块,LLM-driven,**fold-only**:digest topic tree 的密度折叠 | -| **scheduler** | L5 Runtime 内部触发器:按 cron + L2 自治状态阈值拉起 background L4 模块(notifier / digester / maintainer) | +| **scheduler** | L5 Runtime 内部触发器:按 cron + L2 自治状态阈值拉起 background L4 模块(notifier / dreamer / maintainer) | | **Topic tree** | digest/ 的心智模型:文件夹 = 中间节点,文件 = 叶子 | | **Fold(密度折叠)** | maintainer 唯一操作:把过密叶子归簇到新子中间节点 + 写高密度摘要 | | **L3 原子工具** | 基础(create/append/edit/read/write/move/delete/list/stat,直 fs)+ 高级(search/traverse/frontmatter,走 L2)两组 | diff --git a/reme4/components/application_context.py b/reme4/components/application_context.py index 4b482b7b..611c188a 100644 --- a/reme4/components/application_context.py +++ b/reme4/components/application_context.py @@ -1,7 +1,7 @@ """Application context: shared state container for components, jobs, and service.""" from concurrent.futures import ThreadPoolExecutor -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from ..enumeration import ComponentEnum from ..schema import ApplicationConfig @@ -29,3 +29,9 @@ class ApplicationContext: self.components: dict[ComponentEnum, dict[str, "BaseComponent"]] = {} self.jobs: dict[str, "BaseJob"] = {} self.thread_pool: ThreadPoolExecutor | None = None + # Side-channel for service/transport-specific objects that don't fit + # the shared component/job model — e.g. MCPService publishes a + # ChannelSink under "channel_sink" so MCP-specific steps + # (claim_channel, channel_notify) can find it. Keep keys narrow: + # if a value is needed across services, promote it to a typed field. + self.metadata: dict[str, Any] = {} diff --git a/reme4/components/job/background_job.py b/reme4/components/job/background_job.py index fa623d78..fc62662a 100644 --- a/reme4/components/job/background_job.py +++ b/reme4/components/job/background_job.py @@ -36,11 +36,13 @@ class BackgroundJob(BaseJob): backoff_cap: float = 60.0, close_timeout: float = 5.0, attempt_reset_after: float = 60.0, - enable_serve: bool = False, use_thread_pool: bool = False, **kwargs, ): - super().__init__(enable_serve=enable_serve, **kwargs) + # Background jobs are long-running loops, not request-shaped callables — + # forced off so they never get registered as service tools. + kwargs.pop("enable_serve", None) + super().__init__(enable_serve=False, **kwargs) self.supervisor: bool = supervisor self.backoff_base: float = backoff_base self.backoff_cap: float = backoff_cap diff --git a/reme4/components/service/mcp_service.py b/reme4/components/service/mcp_service.py index 9203aff9..99287ccd 100644 --- a/reme4/components/service/mcp_service.py +++ b/reme4/components/service/mcp_service.py @@ -1,23 +1,117 @@ -"""MCP (Model Context Protocol) service: exposes jobs as MCP tools.""" +"""MCP service: expose jobs as MCP tools. +Channel binding (the `` +push from background steps to a specific Claude Code window) is uniform +across transports: a single `ChannelSink` lives on +`ApplicationContext.metadata["channel_sink"]`, unbound at startup, and any +client calling the `claim_channel` MCP tool binds itself as the recipient +via `fastmcp.server.dependencies.get_context().session`. Last-claim-wins. + +Under stdio (one client per server process) the client should claim once +after init; until then channel events drop silently. Under shared +streamable-http / sse the human picks which window receives events. + +``ChannelSink`` is colocated here because it is the runtime mechanism +behind this service's channel feature — pushes ``notifications/claude/channel`` +frames to the bound MCP session. Lossy by design: not bound → no-op; +``send_message`` raises → log warning, swallow (failed notifications must +not surface as ingest failures). Uses ``ServerSession.send_message`` +(low-level raw frame) instead of ``send_notification`` because the latter +validates against a closed ``ServerNotification`` RootModel union that +does not include ``notifications/claude/channel`` — Pydantic rejects +custom methods. Meta keys are filtered to ``[A-Za-z0-9_]+``: Claude Code +silently drops keys with hyphens / other chars when projecting onto +```` attrs. +""" + +import re from typing import TYPE_CHECKING from fastmcp import FastMCP from fastmcp.server.server import Transport from fastmcp.tools import FunctionTool +from mcp.shared.message import SessionMessage +from mcp.types import JSONRPCMessage, JSONRPCNotification from .base_service import BaseService from ..component_registry import R from ..job import BaseJob, StreamJob from ...constants import REME_DEFAULT_HOST, REME_DEFAULT_PORT +from ...utils import get_logger if TYPE_CHECKING: + from mcp.server.session import ServerSession + from ...application import Application +_IDENT_RE = re.compile(r"^[A-Za-z0-9_]+$") +_CHANNEL_METHOD = "notifications/claude/channel" + + +class ChannelSink: + """Hold a bound MCP ``ServerSession`` and forward channel events to it.""" + + def __init__(self) -> None: + self._session: "ServerSession | None" = None + self._logger = get_logger() + + def bind(self, session: "ServerSession") -> None: + """Set ``session`` as the recipient for subsequent ``emit`` calls (last-claim-wins).""" + self._session = session + + def unbind(self) -> None: + """Drop the bound session; future ``emit`` calls become no-ops until rebind.""" + self._session = None + + async def emit(self, content: str, meta: dict[str, str] | None = None) -> None: + """Send one channel notification; no-op if unbound, log+swallow on transport failure.""" + session = self._session + if session is None: + return + + clean_meta = {k: str(v) for k, v in (meta or {}).items() if _IDENT_RE.match(k)} + message = SessionMessage( + JSONRPCMessage( + JSONRPCNotification( + jsonrpc="2.0", + method=_CHANNEL_METHOD, + params={"content": content, "meta": clean_meta}, + ), + ), + ) + + try: + await session.send_message(message) + except Exception as exc: + self._logger.warning(f"ChannelSink: send_message failed ({type(exc).__name__}: {exc})") + + +_CHANNEL_INSTRUCTIONS = ( + "Events from the reme channel arrive as\n" + ' \n' + " added|modified|deleted: \n" + " ...\n" + " \n" + "The vault watcher fires one event per debounced batch of live changes " + "under daily/, digest/, and resource/ (initial-scan diffs at startup are " + "intentionally NOT replayed).\n" + "\n" + "Events are delivered ONLY to the MCP session that called the " + "`claim_channel` tool last. Call it once per Claude Code window that " + "should receive vault-change notifications.\n" + "\n" + "When new files appear under daily/ or resource/, treat it as a suggestion " + "to run `/dream ` on each new path -- unless the user is mid-task and " + "would be interrupted, in which case acknowledge in one line and continue. " + "For changes under digest/ (which /dream itself writes), just acknowledge; " + "do not re-dream them. For deletes or modifies elsewhere, just acknowledge." +) + + @R.register("mcp") class MCPService(BaseService): - """Expose non-stream jobs as MCP tools over stdio, SSE, or other supported transports.""" + """Expose non-stream jobs as MCP tools over stdio, SSE, or streamable-http.""" def __init__( self, @@ -34,14 +128,16 @@ class MCPService(BaseService): # ----- BaseService contract ------------------------------------------ def build_service(self, app: "Application") -> None: - """Create the FastMCP server with an app-managed lifespan.""" + """Construct the FastMCP server and publish an unbound ChannelSink.""" + app.context.metadata["channel_sink"] = ChannelSink() self.service = FastMCP( name=app.config.app_name, + instructions=_CHANNEL_INSTRUCTIONS, lifespan=self._lifespan(app, self.host, self.port), ) def add_job(self, job: BaseJob) -> None: - """Register a non-stream job as an MCP tool; StreamJobs are skipped (not supported).""" + """Register a non-stream job as an MCP tool; StreamJobs are unsupported.""" if isinstance(job, StreamJob): return @@ -59,7 +155,7 @@ class MCPService(BaseService): ) def start_service(self, app: "Application") -> None: - """Run the MCP server; bind host/port only when the transport is network-based.""" + """Run the MCP server; bind host/port only for network transports.""" transport_kwargs: dict = {} if self.transport != "stdio": transport_kwargs["host"] = self.host diff --git a/reme4/config/default.yaml b/reme4/config/default.yaml index 837b9a53..782d5bcb 100644 --- a/reme4/config/default.yaml +++ b/reme4/config/default.yaml @@ -13,7 +13,7 @@ jobs: watch_paths: [ "daily", "digest" ] suffix_filters: [ "md" ] steps: - - backend: scan_changes_step + - backend: scan_store_changes_step - backend: update_index_step persist: true - backend: watch_changes_step @@ -108,6 +108,26 @@ jobs: expand_links: true max_links_per_direction: 10 + node_search: + backend: base + description: "Digest node recall — given a candidate abstraction's name+description, surface existing digest nodes similar enough to either dedup against or link to as related." + parameters: + type: object + properties: + query: + type: string + description: "search query" + limit: + type: integer + description: "max digest nodes to return" + default: 20 + required: + - query + steps: + - backend: node_search_step + vector_weight: 0.7 + candidate_multiplier: 5.0 + daily_create: backend: base description: "Provision a session note under a daily folder: daily//.md or daily/.md" @@ -385,7 +405,7 @@ jobs: required: - path steps: - - backend: dreamer_step + - backend: dream_step auto-dream: backend: base @@ -402,7 +422,7 @@ jobs: description: "caller guidance passed through to each per-file dream" default: "" steps: - - backend: cron_dreamer_step + - backend: auto_dream_step auto_memory: backend: base @@ -456,19 +476,21 @@ components: stream: true context_size: 200000 max_retries: 3 - retry_delay: 1.0 credential: api_key: ${LLM_API_KEY:-} base_url: ${LLM_BASE_URL:-https://dashscope.aliyuncs.com/apps/anthropic} parameters: - max_tokens: 100000 + max_tokens: 65536 thinking_enable: true - thinking_budget: 38000 file_graph: default: backend: local + file_catalog: + default: + backend: local + file_parser: linked: backend: linked diff --git a/reme4/steps/__init__.py b/reme4/steps/__init__.py index 82228673..fc492938 100644 --- a/reme4/steps/__init__.py +++ b/reme4/steps/__init__.py @@ -7,8 +7,9 @@ from .common.help import HelpStep from .common.llm_demo import LLMDemoStep from .common.stream_demo import StreamDemoStep1, StreamDemoStep2 from .common.version import VersionStep +from .evolve.auto_dream import AutoDreamStep from .evolve.auto_memory import AutoMemoryStep -from .evolve.auto_dream import CronDreamer, Dreamer +from .evolve.dream import DreamStep from .file_io.daily_create import DailyCreateStep from .file_io.daily_list import DailyListStep from .file_io.daily_reindex import DailyReindexStep @@ -23,8 +24,11 @@ from .file_io.read import ReadStep from .file_io.read_image import ReadImageStep from .file_io.stat import StatStep from .file_io.write import WriteStep +from .index.channel_notify import ChannelNotifyStep +from .index.claim_channel import ClaimChannelStep from .index.clear_and_scan import ClearAndScanStep -from .index.scan_changes import ScanChangesStep +from .index.node_search import NodeSearchStep +from .index.scan_changes import ScanCatalogChangesStep, ScanStoreChangesStep from .index.search import SearchStep from .index.traverse import TraverseStep from .index.update_catalog import UpdateCatalogStep @@ -65,16 +69,20 @@ __all__ = [ "FrontmatterReadStep", "FrontmatterUpdateStep", # index + "ChannelNotifyStep", + "ClaimChannelStep", "ClearAndScanStep", - "ScanChangesStep", + "NodeSearchStep", + "ScanCatalogChangesStep", + "ScanStoreChangesStep", "SearchStep", "TraverseStep", "UpdateCatalogStep", "UpdateIndexStep", "WatchChangesStep", # evolve (dream) - "CronDreamer", - "Dreamer", + "AutoDreamStep", + "DreamStep", # transfer "DownloadStep", "IngestStep", diff --git a/reme4/steps/base_step.py b/reme4/steps/base_step.py index 17587e51..ec507114 100644 --- a/reme4/steps/base_step.py +++ b/reme4/steps/base_step.py @@ -5,8 +5,9 @@ from abc import abstractmethod, ABC from pathlib import Path from typing import TypeVar, TYPE_CHECKING +from agentscope.message import TextBlock from agentscope.model import ChatModelBase -from agentscope.tool import Toolkit, FunctionTool +from agentscope.tool import Toolkit, FunctionTool, ToolChunk from ..components.base_component import ComponentMixin from ..components.file_parser import BaseFileParser @@ -132,8 +133,8 @@ class BaseStep(ComponentMixin, ABC): # Load class-level prompts first, then overlay caller-provided overrides. # Walk MRO in reverse so most-derived class wins; subclasses without their - # own YAML inherit prompts from their parent (e.g. CronDreamer inherits - # dreamer.yaml from Dreamer). + # own YAML inherit prompts from their parent (e.g. AutoDreamStep inherits + # dream.yaml from DreamStep). self.prompt = PromptHandler(language=self.language) for cls in reversed(self.__class__.__mro__): self.prompt.load_prompt_by_class(cls) @@ -217,9 +218,12 @@ class BaseStep(ComponentMixin, ABC): if job is None: raise RuntimeError(f"Job {job_name} not found") - async def run_job(**_kwargs) -> str: + async def run_job(**_kwargs) -> ToolChunk: response = await job(**{**_kwargs, **kwargs}) - return response.answer + return ToolChunk( + content=[TextBlock(text=str(response.answer))], + state="success" if response.success else "error", + ) tool = FunctionTool( func=run_job, diff --git a/reme4/steps/evolve/auto_dream.py b/reme4/steps/evolve/auto_dream.py index 09511bcf..d2efd6e3 100644 --- a/reme4/steps/evolve/auto_dream.py +++ b/reme4/steps/evolve/auto_dream.py @@ -1,547 +1,154 @@ -"""Dreamer — auto-dream's create_or_update step. +"""AutoDreamStep — daily-tick wrapper around :class:`DreamStep`. -Reads one daily-event note or resource file at the given vault-relative -``path``, identifies the ABSTRACTIONS the material teaches in Phase 1 -(each tagged with one of the three buckets), then in Phase 2 makes -ONE cognitive write decision (CREATE or one of the three UPDATE -flavors: CORROBORATE / REFINE / CORRECT) per abstraction using a -**bucket-specific** integrate prompt. +Each tick scans today's two surfaces under ``/``: -**Digest is the abstract memory layer** — raw details stay in the -material; digest holds the principle, pattern, or precedent worth -recalling once the specifics fade. Provenance wikilinks -(``derived_from::``) let readers drill back down to the source. +* ``/.md`` — the day-index file (auto-rebuilt rollup + of today's notes); included first so day-level abstractions land + before per-event details. +* ``//**/*.md`` — event notes for the day. -Pipeline (external loop in Python, two distinct ReAct agent invocations, -**light Phase 1 / heavy Phase 2**): +The diff vs ``file_catalog`` follows the same shape as +:class:`ScanCatalogChangesStep`: build ``existing`` (on-disk ``rel → mtime``) +and ``indexed`` (catalog ``rel → mtime``, restricted to today's +prefix so we never disturb entries from other days), then: - execute(): - units, _ = _extract(material_blob) # 1× ReAct: identify abstractions - # agent emits ExtractedUnits - # ({units: [{name, bucket, summary}, ...]}) - for unit in units: # Python loop, K iterations - _integrate_unit(unit) # 1× ReAct per abstraction, dispatched - # to integrate_system_prompt_; - # recalls cross-bucket, decides write, - # uses canonical write/edit/frontmatter_update tools. +* ``existing`` keys not in ``indexed`` → **added**, dream +* mtime mismatch → **modified**, dream +* ``indexed`` keys not in ``existing`` → **deleted**, drop from catalog +* mtime match → **unchanged**, skip -The bucket vocabulary is hard-coded (:data:`BUCKETS`) — three buckets, -each with a dedicated Phase 2 prompt: +After dreaming, successful (and Phase 1 vacuously-skipped) files +upsert their current ``st_mtime`` so the next tick re-dreams only +what actually changed. Failures leave the catalog untouched and will +be retried on the next tick. -* ``procedure`` — how-to-do-X: steps, methods, recipes, workflows. -* ``personal`` — user/team specific: identity, preferences, - conventions, things they avoid. -* ``wiki`` — general knowledge: definitions, principles, - observations, decisions-as-precedent. Default catch-all. +The two writers (``WatchDreamStep`` event-driven, ``AutoDreamStep`` +catch-up scan) share the protocol: upsert ``(path, st_mtime)`` on +success. Last-writer-wins is fine — same path + same content yields +the same mtime, so they cannot disagree on what's "done". -There is no SKIP outcome in Phase 2: Phase 1 is the gate for "not -worth memorizing"; anything reaching Phase 2 warrants a write. +Cron scheduling itself is out of scope; this step is the unit of +work — invoke it from a system cron, ``reme auto-dream date=...``, +or any other catch-up trigger when ``auto_dream_loop`` missed a file +(e.g. process crashed before the watcher fired). -Phase 2 uses the **canonical** ``write`` / ``edit`` jobs (no -constrained variants). Bucket placement and edge conservation are -prompt-level discipline; the tools themselves perform no path-shape -or conservation validation. +Inputs (RuntimeContext): + date (str, optional): YYYY-MM-DD to scan. Defaults to today + in the dreamer's timezone. + hint (str, optional): passed through to each per-file dream. -Invocation form (CLI / MCP): - reme dream path=daily/2026-05-28/auth-refactor/auth-refactor.md - reme dream path=resource/2026-05-28/spec.pdf hint="focus on auth" +Step kwargs (from yaml ``backend: auto_dream_step``): + persist (bool, default True): when True, ``file_catalog.dump()`` + is called after the batch so progress survives a restart. + +The outer loop reuses ``DreamStep``'s prompt mounting via MRO — no +separate YAML; ``dream.yaml`` is found through the parent class. """ -import datetime -import zoneinfo from pathlib import Path -from typing import Literal -from agentscope.agent import Agent -from agentscope.message import Msg, TextBlock -from agentscope.permission import PermissionContext, PermissionMode -from agentscope.state import AgentState -from agentscope.tool import Toolkit from pydantic import BaseModel, Field -from ..base_step import BaseStep +from .dream import DreamStep, DreamResult +from ..base_step import Ref from ...components import R +from ...components.file_catalog import BaseFileCatalog +from ...enumeration import ComponentEnum +from ...schema import FileNode -# Hard-coded bucket vocabulary. Phase 1 classifies each sub-unit into -# one of these; Phase 2 dispatches to the bucket-specific prompt. -# Order matters for prompt rendering — keep procedure/personal/wiki. -BUCKETS: tuple[str, ...] = ("procedure", "personal", "wiki") - -# Bucket = Literal of BUCKETS. Pydantic Literal must be a static type; -# update both BUCKETS and Bucket together if the vocabulary changes. -Bucket = Literal["procedure", "personal", "wiki"] - - -_EXTRACT_TOOLS: tuple[str, ...] = ("read",) - -_INTEGRATE_TOOLS: tuple[str, ...] = ( - # read - "search", - "traverse", - "read", - "frontmatter_read", - # write - "write", - "edit", - "frontmatter_update", -) - - -def _pack_material(file_store, path: str) -> str: - """Render one daily-event note or resource file into a prompt block.""" - try: - absolute = (Path(file_store.vault_path or ".") / path).resolve() - except Exception as e: - return f"### {path}\n(error resolving path: {type(e).__name__}: {e})\n" - - if not absolute.is_file(): - return f"### {path}\n(file not found)\n" - - try: - return f"### {path}\n{absolute.read_text(encoding='utf-8')}\n" - except Exception as e: - return f"### {path}\n(error reading: {type(e).__name__}: {e})\n" - - -class MemoryUnit(BaseModel): - """One memory sub-unit identified by Phase 1's structured output.""" - - name: str = Field( - description=( - "Short kebab-case identifier for the abstraction " - "(e.g. 'jwt-rotation-decision', 'pr-size-pref'). " - "Agent-internal handle — NOT the eventual digest slug; " - "Phase 2 picks the actual filing path." - ), - ) - bucket: Bucket = Field( - description=( - "Which bucket this abstraction belongs in — Phase 2 dispatches " - "to a bucket-specific prompt based on this. Pick exactly one: " - "`procedure` (how-to-do-X — steps, methods, recipes, workflows), " - "`personal` (user/team-specific — identity, preferences, " - "conventions, things they avoid), `wiki` (general knowledge — " - "definitions, principles, observations, decisions-as-precedent; " - "default catch-all when nothing else fits)." - ), - ) - summary: str = Field( - description=( - "1-2 sentences naming the abstraction AND pointing at where " - "in the material the supporting evidence lives " - "(e.g. 'short-credential compliance drives auth cadence; " - "illustrated by the 30→24h decision in the 'Decision' section " - "+ the SOC2 CC6.1 criticism in the 'Observation' section')." - ), - ) - - -class ExtractedUnits(BaseModel): - """Structured output emitted by Phase 1's extract agent.""" - - units: list[MemoryUnit] = Field( - default_factory=list, - description=( - "Memory sub-units identified in the material — orthogonal " - "abstractions (principles / patterns / precedents) worth " - "lifting into long-term memory. Each is tagged with its " - "bucket. Empty list = nothing worth lifting (Phase 2 is skipped)." - ), - ) - - -def _render_outcome_line(unit_name: str, bucket: str, o: "IntegrateOutcome") -> str: - """Format one IntegrateOutcome as a one-line summary entry.""" - body = f"{o.action} {o.target_path}" - if o.note: - body += f" — {o.note}" - return f"[{unit_name}/{bucket}] {body}" - - -class IntegrateOutcome(BaseModel): - """Structured outcome reported by Phase 2 for one sub-unit.""" - - action: Literal["CREATE", "CORROBORATE", "REFINE", "CORRECT"] = Field( - description=( - "Outcome of the write decision for this sub-unit. Phase 1 already " - "filtered out non-abstractions, so every sub-unit reaching you " - "warrants a write — pick the matching fine-grained action: " - "`CREATE` — brand-new digest node (recall returned no node " - "covering this abstraction); even thin first-encounter seeds go " - "here, they grow via CORROBORATE / REFINE on later passes. " - "`CORROBORATE` (most common when a covering node exists) — " - "provenance append + optional wording strengthening; the " - "abstraction already covers this material. `REFINE` — covering " - "node exists but the material reveals nuance, scope, or edge " - "cases the abstraction under-specified. `CORRECT` — covering " - "node exists but the material contradicts it; tighten the " - "abstraction or annotate the contradiction inline." - ), - ) - target_path: str = Field( - description=("The digest path you wrote to — must match what your `write` / " "`edit` call(s) targeted."), - ) - note: str = Field( - default="", - description=( - "Optional ONE short line, ≤ 200 chars, no newlines, summarizing " - "what landed (e.g. 'extended scope to also cover X'). Do NOT " - "dump recall summaries, search results, internal reasoning, or " - "transcripts here — those belong in the ReAct trace, not the " - "outcome note." - ), - ) - - -class DreamResult(BaseModel): - """Outcome of one dreamer invocation. - - Per-tool audit lives in the toolkit layer (not exposed back to the - orchestrator). Structured outcome here is the input path the call - processed, the memory sub-units the agent declared in Phase 1, and - what got created / updated in Phase 2. - """ - - used_llm: bool = False - skipped: bool = False - path: str = "" - units: list[dict] = Field(default_factory=list) - nodes_created: list[str] = Field(default_factory=list) - nodes_updated: list[str] = Field(default_factory=list) - summary: str = "" - error: str = "" - - -@R.register("dreamer_step") -class Dreamer(BaseStep): - """auto-dream create_or_update step. - - Inputs (from RuntimeContext): - path (str, required): vault-relative path of one - daily-event note or resource file to dream over. Pass - empty string to no-op. - hint (str, optional): caller guidance to the LLM - (e.g. "focus on the auth-related decisions"). - - Output (written to context.response.answer): - ``DreamResult`` JSON in ``metadata``; LLM summary in ``answer``. - - CLI / MCP form: - reme dream path=daily/2026-05-28/auth-refactor/auth-refactor.md - """ - - def __init__( - self, - toolkit: Toolkit | None = None, - timezone: str | None = None, - **kwargs, - ): - super().__init__(**kwargs) - self.toolkit = toolkit - self.timezone = timezone - - def _now(self) -> datetime.datetime: - if self.timezone: - try: - return datetime.datetime.now(zoneinfo.ZoneInfo(self.timezone)) - except Exception as e: - self.logger.error(f"Invalid timezone: {self.timezone}, error={e}") - return datetime.datetime.now() - - def _vault_dir(self) -> Path: - vr = getattr(self.file_store, "vault_path", None) - return Path(vr).resolve() if vr else Path.cwd().resolve() - - def _llm_available(self) -> bool: - try: - return self.llm is not None - except Exception: - return False - - def _build_extract_toolkit(self) -> Toolkit: - """Read-only toolkit for the extract agent. Sub-units come back via - :class:`ExtractedUnits` structured output, not via a tool call.""" - toolkit = Toolkit() - for job_name in _EXTRACT_TOOLS: - self.add_as_tool(toolkit, job_name) - return toolkit - - def _build_integrate_toolkit(self) -> Toolkit: - """Full read + canonical write/edit/frontmatter_update toolkit for - the integrate agent. All tools are registered via :meth:`add_as_tool` - — same as every other step in this codebase. Outcome tracking is - driven by the agent's :class:`IntegrateOutcome` structured emission, - not by per-tool callbacks.""" - toolkit = self.toolkit or Toolkit() - for job_name in _INTEGRATE_TOOLS: - self.add_as_tool(toolkit, job_name) - return toolkit - - async def _extract(self, material_blob: str, hint: str, vault_dir: Path) -> tuple[list[dict], str]: - """Phase 1: one ReAct invocation — read material + emit ExtractedUnits. - - Returns ``(units, llm_summary)`` where ``units`` is the cleaned - sub-unit list (each entry has ``name`` / ``bucket`` / ``summary``) - and ``llm_summary`` is whatever free-form text the agent produced - alongside its structured emission. - """ - toolkit = self._build_extract_toolkit() - agent = Agent( - name="reme_dreamer_extract", - model=self.llm, - system_prompt=self.prompt_format( - "extract_system_prompt", - vault_dir=str(vault_dir), - buckets=", ".join(BUCKETS), - ), - toolkit=toolkit, - state=AgentState( - permission_context=PermissionContext( - mode=PermissionMode.BYPASS, - ), - ), - ) - user_message = self.prompt_format( - "extract_user_message", - today=self._now().strftime("%Y-%m-%d"), - hint=hint or "(none)", - material_blob=material_blob, - ) - msg = await agent.reply( - Msg(name="reme", role="user", content=[TextBlock(text=user_message)]), - ) - - structured_resp = await self.llm.generate_structured_output( - agent.state.context, - structured_model=ExtractedUnits, - ) - meta = structured_resp.content if isinstance(structured_resp.content, dict) else {} - cleaned: list[dict] = [] - for raw in meta.get("units") or []: - if not isinstance(raw, dict): - continue - name = str(raw.get("name") or "").strip() - summary = str(raw.get("summary") or "").strip() - bucket = str(raw.get("bucket") or "").strip() - if not name or not summary: - continue - if bucket not in BUCKETS: - # Defensive: structured_model should already reject this, - # but if it slips through we route to wiki (the catch-all). - self.logger.warning( - f"[{self.name}] unit {name!r} emitted bucket {bucket!r} " - f"not in {list(BUCKETS)}; routing to 'wiki'", - ) - bucket = "wiki" - cleaned.append({"name": name, "summary": summary, "bucket": bucket}) - return cleaned, (msg.get_text_content() or "").strip() - - async def _integrate_unit(self, unit: dict, material_blob: str, hint: str, vault_dir: Path) -> IntegrateOutcome: - """One ReAct invocation per memory sub-unit, dispatched to the - bucket-specific system prompt. Returns the parsed - :class:`IntegrateOutcome` reported by the agent — that's the - single source of truth for what got written (action + - target_path).""" - bucket = unit.get("bucket") or "wiki" - toolkit = self._build_integrate_toolkit() - digest_dir = getattr(self.app_context.app_config, "digest_dir", "") - agent = Agent( - name=f"reme_dreamer_integrate_{unit.get('name', 'unit')}", - model=self.llm, - system_prompt=self.prompt_format( - f"integrate_system_prompt_{bucket}", - vault_dir=str(vault_dir), - digest_dir=digest_dir, - bucket=bucket, - ), - toolkit=toolkit, - state=AgentState( - permission_context=PermissionContext( - mode=PermissionMode.BYPASS, - ), - ), - ) - user_message = self.prompt_format( - "integrate_user_message", - hint=hint or "(none)", - unit_name=unit.get("name", ""), - unit_bucket=bucket, - unit_summary=unit.get("summary", ""), - material_blob=material_blob, - ) - await agent.reply( - Msg(name="reme", role="user", content=[TextBlock(text=user_message)]), - ) - structured_resp = await self.llm.generate_structured_output( - agent.state.context, - structured_model=IntegrateOutcome, - ) - return IntegrateOutcome.model_validate(structured_resp.content) - - async def dream_one(self, path: str, hint: str = "") -> DreamResult: - """Run the full extract + integrate pipeline on one vault-relative - material path. Returns a structured :class:`DreamResult`. Safe to - call repeatedly on the same instance — per-invocation trackers are - reset at the start of each call. Used both by :meth:`execute` - (single file from context) and by :class:`CronDreamer` (loop over - today's materials). - """ - path = (path or "").strip() - hint = (hint or "").strip() - - if not path: - return DreamResult(used_llm=False, skipped=True) - - if not self._llm_available(): - return DreamResult( - used_llm=False, - skipped=True, - path=path, - error="no llm configured; dreaming requires an LLM", - ) - - material_blob = _pack_material(self.file_store, path) - - vault_dir = self._vault_dir() - - # Phase 1 — extract (light). Agent emits ExtractedUnits structured output to commit the - # memory sub-units worth lifting. Each unit carries its own bucket. - self.logger.info(f"[{self.name}] extract phase: path={path!r}") - units, extract_summary = await self._extract(material_blob, hint, vault_dir) - - if not units: - return DreamResult( - used_llm=True, - path=path, - summary=extract_summary or "no memory sub-units declared", - skipped=True, - ) - - unit_handles = ", ".join(f"{u['name']}/{u['bucket']}" for u in units) - self.logger.info(f"[{self.name}] integrate phase: {len(units)} sub-unit(s): {unit_handles}") - - # Phase 2 — integrate, one fresh ReAct per sub-unit, dispatched to - # the bucket-specific system prompt. Python-level loop, not agent - # loop. Each session emits a structured IntegrateOutcome whose - # action + target_path are the source of truth for what landed. - nodes_created: list[str] = [] - nodes_updated: list[str] = [] - per_unit_lines: list[str] = [] - for i, unit in enumerate(units, start=1): - name = unit.get("name", "?") - bucket = unit.get("bucket", "?") - try: - outcome = await self._integrate_unit(unit, material_blob, hint, vault_dir) - except Exception as e: - self.logger.error( - f"[{self.name}] integrate {i}/{len(units)} " - f"(unit={name}, bucket={bucket}) failed: {type(e).__name__}: {e}", - ) - per_unit_lines.append(f"[{name}/{bucket}] FAILED: {type(e).__name__}: {e}") - continue - if outcome.action == "CREATE": - nodes_created.append(outcome.target_path) - else: - nodes_updated.append(outcome.target_path) - per_unit_lines.append(_render_outcome_line(name, bucket, outcome)) - - per_unit_block = "\n".join(per_unit_lines) - summary = ( - f"Declared {len(units)} sub-unit(s) ({unit_handles}); " - f"created {len(nodes_created)}, updated {len(nodes_updated)}.\n" - f"{per_unit_block}" - ) - - return DreamResult( - used_llm=True, - path=path, - units=units, - nodes_created=nodes_created, - nodes_updated=nodes_updated, - summary=summary, - skipped=False, - ) - - async def execute(self): - assert self.context is not None - path: str = (self.context.get("path", "") or "").strip() - hint: str = (self.context.get("hint", "") or "").strip() - - result = await self.dream_one(path, hint) - - if not path: - self.context.response.success = True - self.context.response.answer = "Skipped: no path supplied" - elif result.error: - self.context.response.success = False - self.context.response.answer = f"Error: {result.error}" - elif result.skipped: - self.context.response.success = True - self.context.response.answer = result.summary or "Skipped: no memory sub-units declared" - else: - self.context.response.success = True - self.context.response.answer = result.summary - self.context.response.metadata.update(result.model_dump()) - - -# ============================================================ -# CronDreamer — daily-tick wrapper around Dreamer. -# -# Inherits the per-file pipeline from Dreamer.dream_one and adds the -# outer loop over today's daily/ + resource/ files. Cron scheduling -# itself is out of scope; this step is just the unit of work. -# -# Inputs (RuntimeContext): -# date (str, optional): YYYY-MM-DD to scan. Defaults to today -# in the dreamer's timezone. -# hint (str, optional): passed through to each per-file dream. -# ============================================================ - - -class CronDreamResult(BaseModel): - """Aggregated outcome of one cron tick.""" +class AutoDreamResult(BaseModel): + """Aggregated outcome of one auto-dream tick.""" date: str = "" files_scanned: int = 0 + files_unchanged: int = 0 files_dreamed: int = 0 files_skipped: int = 0 files_failed: int = 0 + files_deleted: int = 0 per_file: list[DreamResult] = Field(default_factory=list) summary: str = "" -@R.register("cron_dreamer_step") -class CronDreamer(Dreamer): - """Loop ``daily//`` + ``resource//`` and dream each file. +@R.register("auto_dream_step") +class AutoDreamStep(DreamStep): + """Scan ``daily/.md`` + ``daily//`` and dream each file + whose ``st_mtime`` doesn't already match its ``file_catalog`` entry.""" - Inherits :data:`auto_dream.yaml` from :class:`Dreamer` (no separate - yaml — there's no extra prompt for the outer loop). - """ + file_catalog: BaseFileCatalog = Ref(BaseFileCatalog, ComponentEnum.FILE_CATALOG) + + def __init__(self, persist: bool = True, **kwargs): + super().__init__(**kwargs) + self.persist: bool = persist async def execute(self): assert self.context is not None date_input: str = (self.context.get("date", "") or "").strip() hint: str = (self.context.get("hint", "") or "").strip() - # daily_dir / resource_dir come from app config — NOT tool params. - # Same convention as daily_create / daily_list / daily_reindex. - # resource_dir may be empty (default) — that just skips the resource scan. + # daily_dir comes from app config — NOT a tool param. Same convention + # as daily_create / daily_list / daily_reindex. cfg = self.app_context.app_config if self.app_context is not None else None daily_dir = (cfg.daily_dir if cfg else "") or "daily" - resource_dir = cfg.resource_dir if cfg else "" today = date_input or self._now().strftime("%Y-%m-%d") vault = self._vault_dir() - files = _scan_today_files(vault, today, daily_dir, resource_dir) + files = _scan_today_files(vault, today, daily_dir) - result = CronDreamResult(date=today, files_scanned=len(files)) + # existing: today's on-disk paths → st_mtime. Insertion order = scan + # order (date.md first, then sorted event notes); preserved through + # the diff so the day-index file is dreamed before per-event notes. + existing: dict[str, float] = {} + for rel in files: + try: + existing[rel] = (vault / rel).stat().st_mtime + except OSError as e: + self.logger.error(f"[{self.name}] stat failed on {rel}: {e}") + + # indexed: catalog entries restricted to today's prefix. Restriction + # is critical — get_nodes() returns all days, but we must only + # consider deletions within today's scan scope. + today_md = f"{daily_dir}/{today}.md" + today_dir = f"{daily_dir}/{today}/" + all_nodes = await self.file_catalog.get_nodes() + indexed: dict[str, float] = { + n.path: n.st_mtime for n in all_nodes if n.path == today_md or n.path.startswith(today_dir) + } + + # Diff — same vocabulary as scan_*_changes_step (added/modified/deleted). + to_dream: list[tuple[str, float]] = [(rel, mt) for rel, mt in existing.items() if indexed.get(rel) != mt] + unchanged: list[str] = [rel for rel, mt in existing.items() if indexed.get(rel) == mt] + to_delete: list[str] = sorted(indexed.keys() - existing.keys()) + + result = AutoDreamResult( + date=today, + files_scanned=len(existing), + files_unchanged=len(unchanged), + files_deleted=len(to_delete), + ) self.logger.info( - f"[{self.name}] cron tick date={today} scanned={len(files)} file(s) under " - f"{daily_dir}/{today}/ + {resource_dir}/{today}/", + f"[{self.name}] auto-dream tick date={today} scanned={len(existing)} " + f"unchanged={len(unchanged)} todo={len(to_dream)} deleted={len(to_delete)} " + f"under {daily_dir}/{today}{{.md,/}}", ) - for rel_path in files: + # Drop catalog entries for files no longer on disk first. Cheap, no + # LLM, and keeps the catalog consistent even if the dream pass below + # errors. + if to_delete: + try: + await self.file_catalog.delete(to_delete) + except Exception as e: # noqa: BLE001 + self.logger.exception( + f"[{self.name}] file_catalog.delete failed: {type(e).__name__}: {e}", + ) + + # Dream + upsert per-file. Single-file granularity means an LLM + # failure on file N doesn't block files N+1..K from advancing their + # catalog mtime. + upsert_nodes: list[FileNode] = [] + for rel_path, mtime in to_dream: try: dr = await self.dream_one(rel_path, hint) except Exception as e: # pylint: disable=broad-except @@ -554,63 +161,74 @@ class CronDreamer(Dreamer): ) result.per_file.append(dr) if dr.error: + # Failures leave the catalog untouched — next tick retries. result.files_failed += 1 - elif dr.skipped: + continue + if dr.skipped: + # Phase 1 said "nothing to extract" — still mark as seen so + # Phase 1 doesn't re-run on every tick. result.files_skipped += 1 else: result.files_dreamed += 1 + upsert_nodes.append(FileNode(path=rel_path, st_mtime=mtime)) - result.summary = _render_cron_summary(result) + if upsert_nodes: + try: + await self.file_catalog.upsert(upsert_nodes) + except Exception as e: # noqa: BLE001 + self.logger.exception( + f"[{self.name}] file_catalog.upsert failed: {type(e).__name__}: {e}", + ) + + if self.persist and (upsert_nodes or to_delete): + try: + await self.file_catalog.dump() + except Exception as e: # noqa: BLE001 + self.logger.exception( + f"[{self.name}] file_catalog.dump failed: {type(e).__name__}: {e}", + ) + + result.summary = _render_auto_dream_summary(result) self.context.response.success = result.files_failed == 0 self.context.response.answer = result.summary self.context.response.metadata.update(result.model_dump()) + return self.context.response -def _scan_today_files( - vault: Path, - today: str, - daily_dir: str, - resource_dir: str, -) -> list[str]: - """Return vault-relative paths of today's daily notes + resource files. +def _scan_today_files(vault: Path, today: str, daily_dir: str) -> list[str]: + """Return vault-relative paths of today's day-index + event notes. * ``/.md`` — the day-index file (auto-rebuilt rollup of all of today's notes). Included first so its day-level abstractions land before the per-event details. * ``//**/*.md`` — event notes for the day, - sorted by path. - * ``//**/*`` — any file type ingested under - today's resource folder. Skipped when ``resource_dir`` is empty. - - Results are sorted for deterministic processing order within each - group; the day-index file leads. + sorted by path for deterministic processing order. """ out: list[str] = [] + if not daily_dir: + return out - if daily_dir: - day_index = vault / daily_dir / f"{today}.md" - if day_index.is_file(): - out.append(str(day_index.relative_to(vault))) - daily_root = vault / daily_dir / today - if daily_root.is_dir(): - for md in sorted(daily_root.rglob("*.md")): - if md.is_file(): - out.append(str(md.relative_to(vault))) + day_index = vault / daily_dir / f"{today}.md" + if day_index.is_file(): + out.append(str(day_index.relative_to(vault))) - if resource_dir: - resource_root = vault / resource_dir / today - if resource_root.is_dir(): - for f in sorted(p for p in resource_root.rglob("*") if p.is_file()): - out.append(str(f.relative_to(vault))) + daily_root = vault / daily_dir / today + if daily_root.is_dir(): + for md in sorted(daily_root.rglob("*.md")): + if md.is_file(): + out.append(str(md.relative_to(vault))) return out -def _render_cron_summary(r: CronDreamResult) -> str: - """One-line header + one line per file with its outcome.""" +def _render_auto_dream_summary(r: AutoDreamResult) -> str: + """One-line header + one line per dreamed file with its outcome. + Unchanged + deleted files are counted in the header only.""" lines = [ - f"[CronDreamer] date={r.date} scanned={r.files_scanned} " - f"dreamed={r.files_dreamed} skipped={r.files_skipped} failed={r.files_failed}", + f"[AutoDreamStep] date={r.date} scanned={r.files_scanned} " + f"unchanged={r.files_unchanged} dreamed={r.files_dreamed} " + f"skipped={r.files_skipped} failed={r.files_failed} " + f"deleted={r.files_deleted}", ] for dr in r.per_file: if dr.error: diff --git a/reme4/steps/evolve/auto_memory.py b/reme4/steps/evolve/auto_memory.py index 4926cec6..7806f885 100644 --- a/reme4/steps/evolve/auto_memory.py +++ b/reme4/steps/evolve/auto_memory.py @@ -6,7 +6,17 @@ decides what to preserve, and writes the note via ``read`` / ``edit`` / ``frontmatter_update`` / ``write`` tools. Inputs (from RuntimeContext): - messages (list[Msg], required): conversation slice to inspect. + messages (list[Msg], optional): conversation slice to inspect. + Mutually exclusive with ``transcript_path``; if both are + provided, ``messages`` wins. + transcript_path (str, optional): absolute path to a Claude Code + transcript JSONL file. When provided (and ``messages`` is + empty), the step parses the file via + :func:`reme4.utils.transcript.load_messages_from_transcript` + and proceeds as if those were the messages. This is what the + ``reme-service`` plugin's PreCompact / SessionEnd hooks pass + in directly via ``type: mcp_tool``, replacing the temporary + spawn-subagent bridge. session_id (str, optional): passed to daily_create to determine the note path. memory_hint (str, optional): caller-supplied hint for the agent. @@ -14,7 +24,7 @@ Inputs (from RuntimeContext): Output (written to context.response): answer: one-line summary from the agent. - metadata: {path, created}. + metadata: {path, created, n_messages, transcript_path?}. """ from agentscope.agent import Agent @@ -26,6 +36,7 @@ from agentscope.tool import Toolkit from ._evolve import format_history, now from ..base_step import BaseStep from ...components import R +from ...utils.transcript import load_messages_from_transcript @R.register("auto_memory_step") @@ -46,24 +57,50 @@ class AutoMemoryStep(BaseStep): async def execute(self): assert self.context is not None - messages: list[Msg] = [self._to_msg(item) for item in self.context.get("messages", [])] + raw_messages = self.context.get("messages") or [] + transcript_path: str = self.context.get("transcript_path", "") or "" session_id: str = self.context.get("session_id", "") memory_hint: str = self.context.get("memory_hint", "") current = now(self.context.get("timezone")) + # If caller passed transcript_path (the canonical Claude Code hook + # input) instead of messages, parse it here so the rest of the step + # stays unchanged. + if not raw_messages and transcript_path: + raw_messages = load_messages_from_transcript(transcript_path) + self.logger.info( + f"[{self.name}] loaded {len(raw_messages)} messages from transcript_path={transcript_path}", + ) + + messages: list[Msg] = [self._to_msg(item) for item in raw_messages] + if not messages: self.context.response.success = True - self.context.response.answer = "Skipped: no messages supplied" + reason = ( + f"Skipped: no messages in transcript_path={transcript_path}" + if transcript_path + else "Skipped: no messages supplied" + ) + self.context.response.answer = reason + self.context.response.metadata.update( + {"n_messages": 0, "transcript_path": transcript_path}, + ) + self.logger.info(f"[{self.name}] skipped: {reason} session_id={session_id!r}") return create_response = await self.run_job("daily_create", session_id=session_id) if not create_response.success: self.context.response.success = False self.context.response.answer = f"daily_create failed: {create_response.answer}" + self.logger.info(f"[{self.name}] daily_create failed session_id={session_id!r}") return note_path: str = create_response.metadata["path"] created: bool = create_response.metadata["created"] + self.logger.info( + f"[{self.name}] note_path={note_path} created={created} " + f"messages={len(messages)} hint={'yes' if memory_hint else 'no'}", + ) toolkit = Toolkit() for job_name in self.agent_tools: @@ -95,4 +132,12 @@ class AutoMemoryStep(BaseStep): self.context.response.success = True self.context.response.answer = (final_msg.get_text_content() or "").strip() - self.context.response.metadata.update({"path": note_path, "created": created}) + self.context.response.metadata.update( + { + "path": note_path, + "created": created, + "n_messages": len(messages), + "transcript_path": transcript_path, + }, + ) + self.logger.info(f"[{self.name}] done note_path={note_path}") diff --git a/reme4/steps/evolve/dream.py b/reme4/steps/evolve/dream.py new file mode 100644 index 00000000..f4218c41 --- /dev/null +++ b/reme4/steps/evolve/dream.py @@ -0,0 +1,499 @@ +"""DreamStep — single-file digest step (auto-dream's create_or_update primitive). + +Reads one daily-event note or resource file at the given vault-relative +``path``, identifies the ABSTRACTIONS the material teaches in Phase 1 +(each tagged with one of the three buckets), then in Phase 2 makes +ONE cognitive write decision (CREATE or one of the three UPDATE +flavors: CORROBORATE / REFINE / CORRECT) per abstraction using a +**bucket-specific** integrate prompt. + +**Digest is the abstract memory layer** — raw details stay in the +material; digest holds the principle, pattern, or precedent worth +recalling once the specifics fade. Provenance wikilinks +(``derived_from::``) let readers drill back down to the source. + +Pipeline (external loop in Python, two distinct ReAct agent invocations, +**light Phase 1 / heavy Phase 2**): + + execute(): + units, _ = _extract(material_blob) # 1× ReAct: identify abstractions + # agent emits ExtractedUnits + # ({units: [{name, bucket, summary}, ...]}) + for unit in units: # Python loop, K iterations + _integrate_unit(unit) # 1× ReAct per abstraction, dispatched + # to integrate_system_prompt_; + # recalls cross-bucket, decides write, + # uses canonical write/edit/frontmatter_update tools. + +The bucket vocabulary is hard-coded (:data:`BUCKETS`) — three buckets, +each with a dedicated Phase 2 prompt: + +* ``procedure`` — how-to-do-X: steps, methods, recipes, workflows. +* ``personal`` — user/team specific: identity, preferences, + conventions, things they avoid. +* ``wiki`` — general knowledge: definitions, principles, + observations, decisions-as-precedent. Default catch-all. + +There is no SKIP outcome in Phase 2: Phase 1 is the gate for "not +worth memorizing"; anything reaching Phase 2 warrants a write. + +Phase 2 uses the **canonical** ``write`` / ``edit`` jobs (no +constrained variants). Bucket placement and edge conservation are +prompt-level discipline; the tools themselves perform no path-shape +or conservation validation. + +Invocation form (CLI / MCP): + reme dream path=daily/2026-05-28/auth-refactor/auth-refactor.md + reme dream path=resource/2026-05-28/spec.pdf hint="focus on auth" +""" + +import datetime +import zoneinfo +from pathlib import Path +from typing import Literal + +from agentscope.agent import Agent +from agentscope.message import Msg, TextBlock +from agentscope.permission import PermissionContext, PermissionMode +from agentscope.state import AgentState +from agentscope.tool import Toolkit +from pydantic import BaseModel, Field + +from ..base_step import BaseStep +from ...components import R + + +# Hard-coded bucket vocabulary. Phase 1 classifies each sub-unit into +# one of these; Phase 2 dispatches to the bucket-specific prompt. +# Order matters for prompt rendering — keep procedure/personal/wiki. +BUCKETS: tuple[str, ...] = ("procedure", "personal", "wiki") + +# Bucket = Literal of BUCKETS. Pydantic Literal must be a static type; +# update both BUCKETS and Bucket together if the vocabulary changes. +Bucket = Literal["procedure", "personal", "wiki"] + + +_EXTRACT_TOOLS: tuple[str, ...] = ("read",) + +_INTEGRATE_TOOLS: tuple[str, ...] = ( + # read — dream uses its own node-level digest search (NOT the + # general chunk-level `search`), specialized for dedup + synapse + # recall. See reme4/steps/index/node_search.py for the rationale. + # NO traverse here: traverse is a retrieve-time subgraph mining + # tool (used by external retrieval agents); dream is a write-time + # candidate recall operation, structurally a different problem. + "node_search", + "read", + "frontmatter_read", + # write + "write", + "edit", + "frontmatter_update", +) + + +# ============================================================ +# Schema (Pydantic models for structured ReAct output) + helpers +# ============================================================ + + +def _pack_material(file_store, path: str) -> str: + """Render one daily-event note or resource file into a prompt block.""" + try: + absolute = (Path(file_store.vault_path or ".") / path).resolve() + except Exception as e: + return f"### {path}\n(error resolving path: {type(e).__name__}: {e})\n" + + if not absolute.is_file(): + return f"### {path}\n(file not found)\n" + + try: + return f"### {path}\n{absolute.read_text(encoding='utf-8')}\n" + except Exception as e: + return f"### {path}\n(error reading: {type(e).__name__}: {e})\n" + + +class MemoryUnit(BaseModel): + """One memory sub-unit identified by Phase 1's structured output.""" + + name: str = Field( + description=( + "Short kebab-case identifier for the abstraction " + "(e.g. 'jwt-rotation-decision', 'pr-size-pref'). " + "Agent-internal handle — NOT the eventual digest slug; " + "Phase 2 picks the actual filing path." + ), + ) + bucket: Bucket = Field( + description=( + "Which bucket this abstraction belongs in — Phase 2 dispatches " + "to a bucket-specific prompt based on this. Pick exactly one: " + "`procedure` (how-to-do-X — steps, methods, recipes, workflows), " + "`personal` (user/team-specific — identity, preferences, " + "conventions, things they avoid), `wiki` (general knowledge — " + "definitions, principles, observations, decisions-as-precedent; " + "default catch-all when nothing else fits)." + ), + ) + summary: str = Field( + description=( + "1-2 sentences naming the abstraction AND pointing at where " + "in the material the supporting evidence lives " + "(e.g. 'short-credential compliance drives auth cadence; " + "illustrated by the 30→24h decision in the 'Decision' section " + "+ the SOC2 CC6.1 criticism in the 'Observation' section')." + ), + ) + + +class ExtractedUnits(BaseModel): + """Structured output emitted by Phase 1's extract agent.""" + + units: list[MemoryUnit] = Field( + default_factory=list, + description=( + "Memory sub-units identified in the material — orthogonal " + "abstractions (principles / patterns / precedents) worth " + "lifting into long-term memory. Each is tagged with its " + "bucket. Empty list = nothing worth lifting (Phase 2 is skipped)." + ), + ) + + +def _render_outcome_line(unit_name: str, bucket: str, o: "IntegrateOutcome") -> str: + """Format one IntegrateOutcome as a one-line summary entry.""" + body = f"{o.action} {o.target_path}" + if o.note: + body += f" — {o.note}" + return f"[{unit_name}/{bucket}] {body}" + + +class IntegrateOutcome(BaseModel): + """Structured outcome reported by Phase 2 for one sub-unit.""" + + action: Literal["CREATE", "CORROBORATE", "REFINE", "CORRECT"] = Field( + description=( + "Outcome of the write decision for this sub-unit. Phase 1 already " + "filtered out non-abstractions, so every sub-unit reaching you " + "warrants a write — pick the matching fine-grained action: " + "`CREATE` — brand-new digest node (recall returned no node " + "covering this abstraction); even thin first-encounter seeds go " + "here, they grow via CORROBORATE / REFINE on later passes. " + "`CORROBORATE` (most common when a covering node exists) — " + "provenance append + optional wording strengthening; the " + "abstraction already covers this material. `REFINE` — covering " + "node exists but the material reveals nuance, scope, or edge " + "cases the abstraction under-specified. `CORRECT` — covering " + "node exists but the material contradicts it; tighten the " + "abstraction or annotate the contradiction inline." + ), + ) + target_path: str = Field( + description=("The digest path you wrote to — must match what your `write` / " "`edit` call(s) targeted."), + ) + note: str = Field( + default="", + description=( + "Optional ONE short line, ≤ 200 chars, no newlines, summarizing " + "what landed (e.g. 'extended scope to also cover X'). Do NOT " + "dump recall summaries, search results, internal reasoning, or " + "transcripts here — those belong in the ReAct trace, not the " + "outcome note." + ), + ) + + +class DreamResult(BaseModel): + """Outcome of one DreamStep invocation. + + Per-tool audit lives in the toolkit layer (not exposed back to the + orchestrator). Structured outcome here is the input path the call + processed, the memory sub-units the agent declared in Phase 1, and + what got created / updated in Phase 2. + """ + + used_llm: bool = False + skipped: bool = False + path: str = "" + units: list[dict] = Field(default_factory=list) + nodes_created: list[str] = Field(default_factory=list) + nodes_updated: list[str] = Field(default_factory=list) + summary: str = "" + error: str = "" + + +# ============================================================ +# DreamStep — the per-file create_or_update step. +# ============================================================ + + +@R.register("dream_step") +class DreamStep(BaseStep): + """auto-dream create_or_update step — one file per call. + + Inputs (from RuntimeContext): + path (str, required): vault-relative path of one + daily-event note or resource file to dream over. Pass + empty string to no-op. + hint (str, optional): caller guidance to the LLM + (e.g. "focus on the auth-related decisions"). + + Output (written to context.response.answer): + ``DreamResult`` JSON in ``metadata``; LLM summary in ``answer``. + + CLI / MCP form: + reme dream path=daily/2026-05-28/auth-refactor/auth-refactor.md + """ + + def __init__( + self, + toolkit: Toolkit | None = None, + timezone: str | None = None, + **kwargs, + ): + super().__init__(**kwargs) + self.toolkit = toolkit + self.timezone = timezone + + def _now(self) -> datetime.datetime: + if self.timezone: + try: + return datetime.datetime.now(zoneinfo.ZoneInfo(self.timezone)) + except Exception as e: + self.logger.error(f"Invalid timezone: {self.timezone}, error={e}") + return datetime.datetime.now() + + def _vault_dir(self) -> Path: + vr = getattr(self.file_store, "vault_path", None) + return Path(vr).resolve() if vr else Path.cwd().resolve() + + def _llm_available(self) -> bool: + try: + return self.llm is not None + except Exception: + return False + + def _build_extract_toolkit(self) -> Toolkit: + """Read-only toolkit for the extract agent. Sub-units come back via + :class:`ExtractedUnits` structured output, not via a tool call.""" + toolkit = Toolkit() + for job_name in _EXTRACT_TOOLS: + self.add_as_tool(toolkit, job_name) + return toolkit + + def _build_integrate_toolkit(self) -> Toolkit: + """Full read + canonical write/edit/frontmatter_update toolkit for + the integrate agent. All tools are registered via :meth:`add_as_tool` + — same as every other step in this codebase. Outcome tracking is + driven by the agent's :class:`IntegrateOutcome` structured emission, + not by per-tool callbacks.""" + toolkit = self.toolkit or Toolkit() + for job_name in _INTEGRATE_TOOLS: + self.add_as_tool(toolkit, job_name) + return toolkit + + async def _extract(self, material_blob: str, hint: str, vault_dir: Path) -> tuple[list[dict], str]: + """Phase 1: one ReAct invocation — read material + emit ExtractedUnits. + + Returns ``(units, llm_summary)`` where ``units`` is the cleaned + sub-unit list (each entry has ``name`` / ``bucket`` / ``summary``) + and ``llm_summary`` is whatever free-form text the agent produced + alongside its structured emission. + """ + toolkit = self._build_extract_toolkit() + agent = Agent( + name="reme_dreamer_extract", + model=self.llm, + system_prompt=self.prompt_format( + "extract_system_prompt", + vault_dir=str(vault_dir), + buckets=", ".join(BUCKETS), + ), + toolkit=toolkit, + state=AgentState( + permission_context=PermissionContext( + mode=PermissionMode.BYPASS, + ), + ), + ) + user_message = self.prompt_format( + "extract_user_message", + today=self._now().strftime("%Y-%m-%d"), + hint=hint or "(none)", + material_blob=material_blob, + ) + msg = await agent.reply( + Msg(name="reme", role="user", content=[TextBlock(text=user_message)]), + ) + + structured_resp = await self.llm.generate_structured_output( + agent.state.context, + structured_model=ExtractedUnits, + ) + meta = structured_resp.content if isinstance(structured_resp.content, dict) else {} + cleaned: list[dict] = [] + for raw in meta.get("units") or []: + if not isinstance(raw, dict): + continue + name = str(raw.get("name") or "").strip() + summary = str(raw.get("summary") or "").strip() + bucket = str(raw.get("bucket") or "").strip() + if not name or not summary: + continue + if bucket not in BUCKETS: + # Defensive: structured_model should already reject this, + # but if it slips through we route to wiki (the catch-all). + self.logger.warning( + f"[{self.name}] unit {name!r} emitted bucket {bucket!r} " + f"not in {list(BUCKETS)}; routing to 'wiki'", + ) + bucket = "wiki" + cleaned.append({"name": name, "summary": summary, "bucket": bucket}) + return cleaned, (msg.get_text_content() or "").strip() + + async def _integrate_unit(self, unit: dict, material_blob: str, hint: str, vault_dir: Path) -> IntegrateOutcome: + """One ReAct invocation per memory sub-unit, dispatched to the + bucket-specific system prompt. Returns the parsed + :class:`IntegrateOutcome` reported by the agent — that's the + single source of truth for what got written (action + + target_path).""" + bucket = unit.get("bucket") or "wiki" + toolkit = self._build_integrate_toolkit() + digest_dir = getattr(self.app_context.app_config, "digest_dir", "") + agent = Agent( + name=f"reme_dreamer_integrate_{unit.get('name', 'unit')}", + model=self.llm, + system_prompt=self.prompt_format( + f"integrate_system_prompt_{bucket}", + vault_dir=str(vault_dir), + digest_dir=digest_dir, + bucket=bucket, + ), + toolkit=toolkit, + state=AgentState( + permission_context=PermissionContext( + mode=PermissionMode.BYPASS, + ), + ), + ) + user_message = self.prompt_format( + "integrate_user_message", + hint=hint or "(none)", + unit_name=unit.get("name", ""), + unit_bucket=bucket, + unit_summary=unit.get("summary", ""), + material_blob=material_blob, + ) + await agent.reply( + Msg(name="reme", role="user", content=[TextBlock(text=user_message)]), + ) + structured_resp = await self.llm.generate_structured_output( + agent.state.context, + structured_model=IntegrateOutcome, + ) + return IntegrateOutcome.model_validate(structured_resp.content) + + async def dream_one(self, path: str, hint: str = "") -> DreamResult: + """Run the full extract + integrate pipeline on one vault-relative + material path. Returns a structured :class:`DreamResult`. Safe to + call repeatedly on the same instance — per-invocation trackers are + reset at the start of each call. Used by :meth:`execute` (single + file from context), :class:`AutoDreamStep` (loop over today's + materials), and :class:`WatchDreamStep` (loop over a change batch). + """ + path = (path or "").strip() + hint = (hint or "").strip() + + if not path: + return DreamResult(used_llm=False, skipped=True) + + if not self._llm_available(): + return DreamResult( + used_llm=False, + skipped=True, + path=path, + error="no llm configured; dreaming requires an LLM", + ) + + material_blob = _pack_material(self.file_store, path) + + vault_dir = self._vault_dir() + + # Phase 1 — extract (light). Agent emits ExtractedUnits structured output to commit the + # memory sub-units worth lifting. Each unit carries its own bucket. + self.logger.info(f"[{self.name}] extract phase: path={path!r}") + units, extract_summary = await self._extract(material_blob, hint, vault_dir) + + if not units: + return DreamResult( + used_llm=True, + path=path, + summary=extract_summary or "no memory sub-units declared", + skipped=True, + ) + + unit_handles = ", ".join(f"{u['name']}/{u['bucket']}" for u in units) + self.logger.info(f"[{self.name}] integrate phase: {len(units)} sub-unit(s): {unit_handles}") + + # Phase 2 — integrate, one fresh ReAct per sub-unit, dispatched to + # the bucket-specific system prompt. Python-level loop, not agent + # loop. Each session emits a structured IntegrateOutcome whose + # action + target_path are the source of truth for what landed. + nodes_created: list[str] = [] + nodes_updated: list[str] = [] + per_unit_lines: list[str] = [] + for i, unit in enumerate(units, start=1): + name = unit.get("name", "?") + bucket = unit.get("bucket", "?") + try: + outcome = await self._integrate_unit(unit, material_blob, hint, vault_dir) + except Exception as e: + self.logger.error( + f"[{self.name}] integrate {i}/{len(units)} " + f"(unit={name}, bucket={bucket}) failed: {type(e).__name__}: {e}", + ) + per_unit_lines.append(f"[{name}/{bucket}] FAILED: {type(e).__name__}: {e}") + continue + if outcome.action == "CREATE": + nodes_created.append(outcome.target_path) + else: + nodes_updated.append(outcome.target_path) + per_unit_lines.append(_render_outcome_line(name, bucket, outcome)) + + per_unit_block = "\n".join(per_unit_lines) + summary = ( + f"Declared {len(units)} sub-unit(s) ({unit_handles}); " + f"created {len(nodes_created)}, updated {len(nodes_updated)}.\n" + f"{per_unit_block}" + ) + + return DreamResult( + used_llm=True, + path=path, + units=units, + nodes_created=nodes_created, + nodes_updated=nodes_updated, + summary=summary, + skipped=False, + ) + + async def execute(self): + assert self.context is not None + path: str = (self.context.get("path", "") or "").strip() + hint: str = (self.context.get("hint", "") or "").strip() + + result = await self.dream_one(path, hint) + + if not path: + self.context.response.success = True + self.context.response.answer = "Skipped: no path supplied" + elif result.error: + self.context.response.success = False + self.context.response.answer = f"Error: {result.error}" + elif result.skipped: + self.context.response.success = True + self.context.response.answer = result.summary or "Skipped: no memory sub-units declared" + else: + self.context.response.success = True + self.context.response.answer = result.summary + self.context.response.metadata.update(result.model_dump()) diff --git a/reme4/steps/evolve/auto_dream.yaml b/reme4/steps/evolve/dream.yaml similarity index 77% rename from reme4/steps/evolve/auto_dream.yaml rename to reme4/steps/evolve/dream.yaml index b91ee2f9..cfe72a18 100644 --- a/reme4/steps/evolve/auto_dream.yaml +++ b/reme4/steps/evolve/dream.yaml @@ -185,20 +185,30 @@ integrate_system_prompt_procedure: | Plain-prose provenance does NOT count (only wikilinks survive future updates). - ## Recall → decide → write + ## Recall → classify → decide → weave - 1. **Recall** — `search` (include verb stems: rotate, migrate, - deploy…) + `traverse depth=2 direction=both` on any hit under - `{digest_dir}/`. Cross-bucket on purpose: an existing match + 1. **Recall** — call `node_search` with whatever queries best + fit the unit (verb stems work well: rotate, migrate, deploy…). + Use `limit=20-30` for broader coverage; one call usually + suffices but feel free to issue more if the unit spans + multiple concept dimensions. Recall feeds BOTH the dedup + judgment (same_abstraction label) and the synapse judgment + (related label). Cross-bucket on purpose: an existing match filed elsewhere beats a duplicate. - 2. **Hit** — `frontmatter_read` triage, then `read` body for - survivors. Same procedure = same trigger + substantially - overlapping steps. New step or one-line nuance is REFINE, - not "different procedure". - 3. **Decide** (exactly one): - - empty hit set → **CREATE** at + 2. **Classify (internal)** — `node_search` returns name + + description inline; triage from that directly. Use `read` + only for the few survivors needing body inspection. Internally + label each candidate (reasoning only, not emitted to output): + - **same_abstraction** — same trigger + substantially + overlapping steps → UPDATE target (new step / nuance is + REFINE, not "different procedure") + - **related** — adjacent procedure / sub-step / failure-mode + cross-ref → synapse wikilink in body + - **unrelated** — drop + 3. **Decide** (exactly one same_abstraction action): + - no same_abstraction hit → **CREATE** at `{digest_dir}/procedure/.md`. - - hit non-empty → **UPDATE** the best match: + - same_abstraction hit → **UPDATE** the best match: - **CORROBORATE** — same procedure observed again; append `derived_from::`, optionally strengthen wording ("consistently used across N runs"). @@ -208,6 +218,11 @@ integrate_system_prompt_procedure: | - **CORRECT** — wrong order, missing critical step, bad outcome; tighten or annotate inline (`> note: contradicted by [[new-material]] — `). + 4. **Synapse weave** (both CREATE and UPDATE) — weave every + `related` candidate from step 2 into the body as `[[Y.md]]`. + CREATE: woven from the start. UPDATE: additive `edit` + (only-add, never drop existing wikilinks). Default to weaving + more, not less — this is the only chance. ## Discipline @@ -280,21 +295,31 @@ integrate_system_prompt_personal: | preference** (not one big node per person) — that's the granularity downstream search will hit. - ## Recall → decide → write + ## Recall → classify → decide → weave - 1. **Recall** — `search` (user/team name + rule keywords: - `user-X-pr-size-pref`, `team-no-friday-deploys`) + - `traverse depth=2 direction=both` on any hit under - `{digest_dir}/`. Personal nodes often link to each other and - to the user's identity node; don't skip traverse. - 2. **Hit** — `frontmatter_read` triage; `read` body for - survivors. Same rule = same actor scope + same governing - principle. A new context where the rule applies is REFINE, - not "different rule". - 3. **Decide** (exactly one): - - empty hit set → **CREATE** at + 1. **Recall** — call `node_search` with whatever queries best + fit the unit (user/team name + rule keywords: + `user-X-pr-size-pref`, `team-no-friday-deploys`). Use + `limit=20-30` for broader coverage; issue more calls if the + rule has multiple scope dimensions. Recall feeds BOTH the + dedup judgment (same_abstraction label) and the synapse + judgment (related label). Personal nodes often link to each + other and to the user's identity node — vector similarity + surfaces those even when literal names differ. + 2. **Classify (internal)** — `node_search` returns name + + description inline; triage from that directly. Use `read` + only for the few survivors needing body inspection. Internally + label each candidate (reasoning only, not emitted to output): + - **same_abstraction** — same actor scope + same governing + principle → UPDATE target (new applicable context is + REFINE, not "different rule") + - **related** — adjacent rule / contrasting preference / + identity node cross-ref → synapse wikilink in body + - **unrelated** — drop + 3. **Decide** (exactly one same_abstraction action): + - no same_abstraction hit → **CREATE** at `{digest_dir}/personal/.md`. - - hit non-empty → **UPDATE** the best match: + - same_abstraction hit → **UPDATE** the best match: - **CORROBORATE** — rule reaffirmed; append `derived_from::`, possibly strengthen certainty ("observed across N independent contexts"). @@ -305,6 +330,11 @@ integrate_system_prompt_personal: | evidence support, OR annotate (`> note: contradicted by [[new-material]] — user now prefers Y`) without arbitrating. + 4. **Synapse weave** (both CREATE and UPDATE) — weave every + `related` candidate from step 2 into the body as `[[Y.md]]`. + CREATE: woven from the start. UPDATE: additive `edit` + (only-add, never drop existing wikilinks). Default to weaving + more, not less — this is the only chance. ## Discipline @@ -366,23 +396,32 @@ integrate_system_prompt_wiki: | - **`derived_from:: [[]]`** — at least one. Plain-prose provenance does NOT count. - ## Recall → decide → write + ## Recall → classify → decide → weave - 1. **Recall** — `search` (noun phrases + common synonyms) + - `traverse depth=2 direction=both` on any hit under - `{digest_dir}/`. Skipping `traverse` is the main failure mode - producing duplicate concept nodes filed under different - terminology — semantically close abstractions often live one - wikilink away from a noisy hit. - 2. **Hit** — `frontmatter_read` triage; `read` body for - survivors. Same abstraction = same definition / principle in - the body, even if wording differs. Slightly different framing - of the same idea is REFINE; outright different concepts are - different nodes. - 3. **Decide** (exactly one): - - empty hit set → **CREATE** at + 1. **Recall** — call `node_search` with whatever queries best + fit the unit (noun phrases + common synonyms). Use + `limit=20-30` for broader coverage; issue more calls when + the abstraction has multiple aspects worth querying + separately. Recall feeds BOTH the dedup judgment + (same_abstraction label) and the synapse judgment + (related label). Vector similarity catches abstractions + filed under different terminology even when surface words + don't overlap. + 2. **Classify (internal)** — `node_search` returns name + + description inline; triage from that directly. Use `read` + only for the few survivors needing body inspection. Internally + label each candidate (reasoning only, not emitted to output): + - **same_abstraction** — same definition / principle in body, + even if wording differs → UPDATE target (slightly different + framing is REFINE; outright different concepts are + different nodes) + - **related** — concept-adjacent / contrasts / sup-/sub- + concept / instance cross-ref → synapse wikilink in body + - **unrelated** — drop + 3. **Decide** (exactly one same_abstraction action): + - no same_abstraction hit → **CREATE** at `{digest_dir}/wiki/.md`. - - hit non-empty → **UPDATE** the best match: + - same_abstraction hit → **UPDATE** the best match: - **CORROBORATE** — principle reaffirmed by new instance; append `derived_from::`, optionally strengthen wording ("consistently observed across N sources" / replace @@ -396,6 +435,11 @@ integrate_system_prompt_wiki: | evidence support, or annotate inline (`> note: contradicted by [[new-material]] — `) without arbitrating. + 4. **Synapse weave** (both CREATE and UPDATE) — weave every + `related` candidate from step 2 into the body as `[[Y.md]]`. + CREATE: woven from the start. UPDATE: additive `edit` + (only-add, never drop existing wikilinks). Default to weaving + more, not less — this is the only chance. ## Discipline @@ -590,19 +634,26 @@ integrate_system_prompt_procedure_zh: | - **`derived_from:: [[]]`** —— 至少一条。纯 散文形式 **不算**(下次 update 时会消失)。 - ## 召回 → 决策 → 写入 + ## 召回 → 内化分类 → 决策 → 织突触 - 1. **召回** —— `search`(带动词词根:rotate / migrate / - deploy…)+ 对 `{digest_dir}/` 下 **任何** 命中跑 - `traverse depth=2 direction=both`。**跨 bucket** 是有意 —— - 就地更新优于复制创建。 - 2. **命中** —— `frontmatter_read` 廉价 triage,幸存者用 - `read` 读完整 body。"同一流程" = 同触发 + 步骤大幅重叠。 - 新增一步 / 细微差异是 REFINE,**不是** 另一个流程。 - 3. **决策**(恰好一种): - - 命中空 → **CREATE** 在 + 1. **召回** —— 调 `node_search`,query 由你自己根据 unit 内容 + 选(带动词词根效果好:rotate / migrate / deploy…)。`limit= + 20-30` 取更宽覆盖;通常一次够,但若 unit 跨多个概念维度可 + 多调几次。召回结果同时服务 dedup(same_abstraction label) + 和 synapse(related label)两类判断。**跨 bucket** 是有意 + —— 就地更新优于复制创建。 + 2. **内化分类** —— `node_search` 已内嵌返回 name + description, + 直接据此 triage。仅对需要看 body 的少数候选用 `read`。 + 对每个候选**内化打 label**(只在思考中分类,不输出): + - **same_abstraction** —— 同触发 + 步骤大幅重叠 → UPDATE + 目标(新增一步 / 细微差异是 REFINE,**不是** 另一个流程) + - **related** —— 邻近流程 / 子步骤 / 失败模式互引 → 织成 + body 内的 synapse wikilink + - **unrelated** —— 丢弃 + 3. **决策**(恰好一个 same_abstraction 动作): + - 无 same_abstraction 命中 → **CREATE** 在 `{digest_dir}/procedure/.md`。 - - 命中非空 → **UPDATE** 最匹配的: + - same_abstraction 命中 → **UPDATE** 最匹配的: - **CORROBORATE** —— 同流程再次出现;加 `derived_from::`, 可选强化措辞("跨 N 次运行一致使用");步骤不动。 - **REFINE** —— 新前置 / 边界 / 失败模式;扩展相关片段, @@ -610,6 +661,10 @@ integrate_system_prompt_procedure_zh: | - **CORRECT** —— 顺序错 / 缺关键步 / 结果不对;收紧或 内联标注(`> note: contradicted by [[new-material]] — <一句话>`)。 + 4. **织突触**(CREATE 与 UPDATE 都要做)—— 把第 2 步所有 + `related` 候选织入 body 作 `[[Y.md]]`。CREATE:写入时一次性 + 织全。UPDATE:additive `edit`(只增不删,绝不丢已有 wikilink)。 + 宁可多织 —— **这是唯一机会**。 ## 纪律 @@ -673,20 +728,28 @@ integrate_system_prompt_personal_zh: | 同一人有多条偏好时,**一条偏好一个节点**(不是一个人一大 节点) —— 这才是下游搜索的粒度。 - ## 召回 → 决策 → 写入 + ## 召回 → 内化分类 → 决策 → 织突触 - 1. **召回** —— `search`(user / team 名 + 规则关键词: - `user-X-pr-size-pref`、`team-no-friday-deploys`)+ 对 - `{digest_dir}/` 下 **任何** 命中跑 - `traverse depth=2 direction=both`。personal 节点常彼此互 - 链并指向用户身份节点;**别跳过 traverse**。 - 2. **命中** —— `frontmatter_read` triage,幸存者 `read` body。 - "同一规则" = 同 actor 范围 + 同支配原则。新增"规则适用情 - 境"是 REFINE,**不是** 另一条规则。 - 3. **决策**(恰好一种): - - 命中空 → **CREATE** 在 + 1. **召回** —— 调 `node_search`,query 由你自己根据 unit 内容 + 选(user / team 名 + 规则关键词:`user-X-pr-size-pref`、 + `team-no-friday-deploys`)。`limit=20-30` 取更宽覆盖;规则 + 若有多个 scope 维度可多调几次。召回结果同时服务 dedup + (same_abstraction label)和 synapse(related label)两类 + 判断。personal 节点常彼此互链并指向用户身份节点 —— vector + 相似度能在字面名不同时也召回这些。 + 2. **内化分类** —— `node_search` 已内嵌返回 name + description, + 直接据此 triage。仅对需要看 body 的少数候选用 `read`。 + 对每个候选**内化打 label**(只在思考中分类,不输出): + - **same_abstraction** —— 同 actor 范围 + 同支配原则 → + UPDATE 目标(新增"规则适用情境"是 REFINE,**不是** 另一 + 条规则) + - **related** —— 邻近规则 / 对比偏好 / 用户身份节点互引 → + 织成 body 内的 synapse wikilink + - **unrelated** —— 丢弃 + 3. **决策**(恰好一个 same_abstraction 动作): + - 无 same_abstraction 命中 → **CREATE** 在 `{digest_dir}/personal/.md`。 - - 命中非空 → **UPDATE** 最匹配的: + - same_abstraction 命中 → **UPDATE** 最匹配的: - **CORROBORATE** —— 规则在新场景再次坐实;加 `derived_from::`,可选强化确定性("跨 N 个独立情境 观察")。 @@ -696,6 +759,10 @@ integrate_system_prompt_personal_zh: | 新旧证据都支持的形式,或内联标注 (`> note: contradicted by [[new-material]] — 用户现在 偏好 Y`)不仲裁。 + 4. **织突触**(CREATE 与 UPDATE 都要做)—— 把第 2 步所有 + `related` 候选织入 body 作 `[[Y.md]]`。CREATE:写入时一次性 + 织全。UPDATE:additive `edit`(只增不删,绝不丢已有 wikilink)。 + 宁可多织 —— **这是唯一机会**。 ## 纪律 @@ -750,19 +817,27 @@ integrate_system_prompt_wiki_zh: | - **`derived_from:: [[]]`** —— 至少一条;纯 散文形式不算。 - ## 召回 → 决策 → 写入 + ## 召回 → 内化分类 → 决策 → 织突触 - 1. **召回** —— `search`(名词短语 + 常见同义词)+ 对 - `{digest_dir}/` 下 **任何** 命中跑 - `traverse depth=2 direction=both`。跳过 `traverse` 是产生 - 重复概念节点(不同 slug 同语义)的主要失败模式 —— 语义相 - 邻的抽象常常就在某个噪音命中的一跳之外。 - 2. **命中** —— `frontmatter_read` triage,幸存者 `read` body。 - "同一抽象" = body 中的定义 / 原则相同(措辞可不同)。同 - 思想的略不同表述是 REFINE;真正不同的概念是不同节点。 - 3. **决策**(恰好一种): - - 命中空 → **CREATE** 在 `{digest_dir}/wiki/.md`。 - - 命中非空 → **UPDATE** 最匹配的: + 1. **召回** —— 调 `node_search`,query 由你自己根据 unit 内容 + 选(名词短语 + 常见同义词)。`limit=20-30` 取更宽覆盖;若 + 抽象本身有多个侧面,值得分别查询时可多调几次。召回结果 + 同时服务 dedup(same_abstraction label)和 synapse + (related label)两类判断。vector 相似度能捕获以不同术语 + 归档的同语义抽象,即便字面词不重叠。 + 2. **内化分类** —— `node_search` 已内嵌返回 name + description, + 直接据此 triage。仅对需要看 body 的少数候选用 `read`。 + 对每个候选**内化打 label**(只在思考中分类,不输出): + - **same_abstraction** —— body 中的定义 / 原则相同(措辞 + 可不同)→ UPDATE 目标(同思想的略不同表述是 REFINE;真正 + 不同的概念是不同节点) + - **related** —— 概念邻近 / 对比 / 上位下位 / 实例互引 → + 织成 body 内的 synapse wikilink + - **unrelated** —— 丢弃 + 3. **决策**(恰好一个 same_abstraction 动作): + - 无 same_abstraction 命中 → **CREATE** 在 + `{digest_dir}/wiki/.md`。 + - same_abstraction 命中 → **UPDATE** 最匹配的: - **CORROBORATE** —— 原则被新实例再坐实;加 `derived_from::`,可选强化措辞("跨 N 个来源一致观 察"、把"似乎"换成"确实");正文实质不变。 @@ -771,6 +846,10 @@ integrate_system_prompt_wiki_zh: | - **CORRECT** —— 事实矛盾或夸大;收紧到新旧证据都支持 的窄形式,或内联标注(`> note: contradicted by [[new-material]] — <一句话>`)不仲裁。 + 4. **织突触**(CREATE 与 UPDATE 都要做)—— 把第 2 步所有 + `related` 候选织入 body 作 `[[Y.md]]`。CREATE:写入时一次性 + 织全。UPDATE:additive `edit`(只增不删,绝不丢已有 wikilink)。 + 宁可多织 —— **这是唯一机会**。 ## 纪律 diff --git a/reme4/steps/file_io/daily_list.py b/reme4/steps/file_io/daily_list.py index 9e9976de..549a6404 100644 --- a/reme4/steps/file_io/daily_list.py +++ b/reme4/steps/file_io/daily_list.py @@ -61,3 +61,4 @@ class DailyListStep(BaseStep): lines = [self._format_note_line(n) for n in notes] self.context.response.answer = "\n".join(lines) if lines else f"No notes found for {day}" self.context.response.metadata.update({"date": day, "count": len(notes)}) + self.logger.info(f"[{self.name}] date={day} notes={len(notes)}") diff --git a/reme4/steps/file_io/daily_reindex.py b/reme4/steps/file_io/daily_reindex.py index 8ef38953..e09cf66e 100644 --- a/reme4/steps/file_io/daily_reindex.py +++ b/reme4/steps/file_io/daily_reindex.py @@ -44,6 +44,7 @@ class DailyReindexStep(BaseStep): self.context.response.success = False self.context.response.answer = f"Error: {refreshed['error']}" self.context.response.metadata.update(refreshed) + self.logger.info(f"[{self.name}] reindex failed error={refreshed['error']!r}") return notes_count = len(refreshed["notes"]) self.context.response.success = True @@ -56,6 +57,10 @@ class DailyReindexStep(BaseStep): "notes_count": notes_count, }, ) + self.logger.info( + f"[{self.name}] date={refreshed['date']} path={refreshed['path']} " + f"created={refreshed['created']} notes={notes_count}", + ) async def execute(self): """Trigger the index rebuild and stamp the response.""" diff --git a/reme4/steps/file_io/delete.py b/reme4/steps/file_io/delete.py index 7d49a152..abac1fea 100644 --- a/reme4/steps/file_io/delete.py +++ b/reme4/steps/file_io/delete.py @@ -47,12 +47,23 @@ class DeleteStep(BaseStep): if "error" in payload: self.context.response.success = False self.context.response.answer = f"Error: {payload['error']}" + self.logger.info(f"[{self.name}] delete failed path={path} error={payload['error']!r}") elif payload.get("is_dir"): self.context.response.success = True self.context.response.answer = f"Deleted directory {path} ({len(payload['deleted_files'])} file(s))" + self.logger.info( + f"[{self.name}] deleted dir path={path} files={len(payload['deleted_files'])} " + f"inbound_files={payload['inbound']['files_touched']} " + f"inbound_links={payload['inbound']['links_total']}", + ) else: self.context.response.success = True self.context.response.answer = f"Deleted {path}" + self.logger.info( + f"[{self.name}] deleted file path={path} " + f"inbound_files={payload['inbound']['files_touched']} " + f"inbound_links={payload['inbound']['links_total']}", + ) self.context.response.metadata.update(payload) async def _delete(self, path: str) -> dict: diff --git a/reme4/steps/file_io/frontmatter_delete.py b/reme4/steps/file_io/frontmatter_delete.py index 2091d515..d1f8bb3a 100644 --- a/reme4/steps/file_io/frontmatter_delete.py +++ b/reme4/steps/file_io/frontmatter_delete.py @@ -59,7 +59,11 @@ class FrontmatterDeleteStep(BaseStep): if "error" in payload: self.context.response.success = False self.context.response.answer = f"Error: {payload['error']}" + self.logger.info(f"[{self.name}] delete failed path={path} error={payload['error']!r}") else: self.context.response.success = True self.context.response.answer = f"Deleted {len(payload['deleted'])} key(s) from {path}" + self.logger.info( + f"[{self.name}] path={path} deleted={payload['deleted']} missing={payload['missing']}", + ) self.context.response.metadata.update(payload) diff --git a/reme4/steps/file_io/frontmatter_read.py b/reme4/steps/file_io/frontmatter_read.py index c9c28e82..26dbcc7a 100644 --- a/reme4/steps/file_io/frontmatter_read.py +++ b/reme4/steps/file_io/frontmatter_read.py @@ -31,11 +31,13 @@ class FrontmatterReadStep(BaseStep): self.context.response.success = False self.context.response.answer = f"Error: {path} not found" self.context.response.metadata.update({"path": path, "exists": False}) + self.logger.info(f"[{self.name}] path={path} exists=False") return if target.suffix != ".md": self.context.response.success = False self.context.response.answer = "Error: not markdown" self.context.response.metadata.update({"path": path, "error": "not markdown"}) + self.logger.info(f"[{self.name}] path={path} error=not_markdown") return try: @@ -44,7 +46,9 @@ class FrontmatterReadStep(BaseStep): self.context.response.success = False self.context.response.answer = f"Error: failed to parse frontmatter in {path}: {exc}" self.context.response.metadata.update({"path": path, "exists": True, "error": str(exc)}) + self.logger.info(f"[{self.name}] path={path} parse_error={exc!r}") return self.context.response.success = True self.context.response.answer = f"Read frontmatter from {path} ({len(meta)} key(s))" self.context.response.metadata.update({"path": path, "exists": True, "frontmatter": meta}) + self.logger.info(f"[{self.name}] path={path} keys={len(meta)}") diff --git a/reme4/steps/file_io/frontmatter_update.py b/reme4/steps/file_io/frontmatter_update.py index d1ad5ad4..fce9a1d0 100644 --- a/reme4/steps/file_io/frontmatter_update.py +++ b/reme4/steps/file_io/frontmatter_update.py @@ -47,7 +47,9 @@ class FrontmatterUpdateStep(BaseStep): if "error" in payload: self.context.response.success = False self.context.response.answer = f"Error: {payload['error']}" + self.logger.info(f"[{self.name}] update failed path={path} error={payload['error']!r}") else: self.context.response.success = True self.context.response.answer = f"Updated {len(metadata)} key(s) on {path}" + self.logger.info(f"[{self.name}] path={path} keys={list(metadata.keys())}") self.context.response.metadata.update(payload) diff --git a/reme4/steps/file_io/move.py b/reme4/steps/file_io/move.py index ffb33c75..de64d050 100644 --- a/reme4/steps/file_io/move.py +++ b/reme4/steps/file_io/move.py @@ -52,9 +52,16 @@ class MoveStep(BaseStep): if "error" in payload: self.context.response.success = False self.context.response.answer = f"Error: {payload['error']}" + self.logger.info(f"[{self.name}] move failed src={src_path} dst={dst_path} error={payload['error']!r}") else: self.context.response.success = True self.context.response.answer = f"Moved {src_path} → {dst_path}" + retarget_info = payload.get("retarget") or {} + self.logger.info( + f"[{self.name}] moved src={src_path} dst={dst_path} src_removed={payload.get('src_removed')} " + f"retarget_files={retarget_info.get('files_touched', 0) if isinstance(retarget_info, dict) else '-'} " + f"retarget_links={retarget_info.get('links_changed', 0) if isinstance(retarget_info, dict) else '-'}", + ) self.context.response.metadata.update(payload) async def _move(self, src_path: str, dst_path: str, overwrite: bool, retarget: bool) -> dict: diff --git a/reme4/steps/file_io/stat.py b/reme4/steps/file_io/stat.py index 97a9dcd8..41a82415 100644 --- a/reme4/steps/file_io/stat.py +++ b/reme4/steps/file_io/stat.py @@ -42,6 +42,7 @@ class StatStep(BaseStep): self.context.response.success = False self.context.response.answer = f"stat: {path} not found" self.context.response.metadata.update({"path": path, "exists": False}) + self.logger.info(f"[{self.name}] path={path} exists=False") return st = target.stat() @@ -69,3 +70,7 @@ class StatStep(BaseStep): self.context.response.success = True self.context.response.answer = answer self.context.response.metadata.update(payload) + self.logger.info( + f"[{self.name}] path={path} type={payload['type']} " + f"size={payload.get('size', '-')} mime={payload.get('mime', '-')}", + ) diff --git a/reme4/steps/index/channel_notify.py b/reme4/steps/index/channel_notify.py new file mode 100644 index 00000000..099c8d56 --- /dev/null +++ b/reme4/steps/index/channel_notify.py @@ -0,0 +1,67 @@ +"""``channel_notify_step`` — push a debounced batch of vault changes as one channel event. + +Designed to be slotted into ``watch_changes_step.dispatch_step`` next to +``update_index_step``: when the watcher emits a batch of changes, this +step forwards a single human-readable summary to +``ApplicationContext.metadata["channel_sink"]``. The Claude Code main +session then sees a ```` +tag and reacts per the server's ``instructions``. + +One event per batch (not per file) — the watcher already de-bounces and +de-duplicates, so a batch is a meaningful "things that changed together" +unit. Putting N events on the wire per batch would multiply session +turns without adding signal. + +No-op (silently) when: + +* ``channel_sink`` is absent from the application context metadata + (e.g. service wasn't an ``MCPService``), so this step is safe in + any pipeline. +* ``context['changes']`` is missing or empty. +* The sink itself has no bound session (no client called ``claim_channel``). +""" + +from pathlib import Path + +from ..base_step import BaseStep +from ...components import R + + +@R.register("channel_notify_step") +class ChannelNotifyStep(BaseStep): + """Forward a batch of vault changes to the Claude Code channel.""" + + async def execute(self): + sink = self.app_context.metadata.get("channel_sink") if self.app_context is not None else None + if sink is None: + return + + changes = (self.context.get("changes", []) if self.context is not None else []) or [] + if not changes: + return + + # Render paths vault-relative so the agent can pass them directly to + # slash commands like /dream . Absolute paths that fall outside + # the vault are left as-is rather than erroring — they shouldn't occur, + # but a stray entry shouldn't kill the event. + vault = self.vault_path + lines: list[str] = [] + for change in changes: + try: + raw = Path(change["path"]) + except (KeyError, TypeError): + continue + try: + shown = str(raw.resolve().relative_to(vault)) + except ValueError: + shown = str(raw) + lines.append(f"{change.get('change', '?')}: {shown}") + + if not lines: + return + + self.logger.info(f"[channel_notify] emit batch count={len(lines)}") + await sink.emit( + content="Vault 变更:\n" + "\n".join(lines), + meta={"kind": "vault_change", "count": str(len(lines))}, + ) diff --git a/reme4/steps/index/claim_channel.py b/reme4/steps/index/claim_channel.py new file mode 100644 index 00000000..756b0afa --- /dev/null +++ b/reme4/steps/index/claim_channel.py @@ -0,0 +1,56 @@ +"""``claim_channel_step`` — let an MCP client elect itself as the ```` recipient. + +The single bind path for every transport (stdio, sse, streamable-http): +this step uses ``fastmcp.server.dependencies.get_context()`` to grab the +current request's ``ServerSession`` and ``ChannelSink.bind`` it. + +Semantics: + +* **Last-claim-wins.** A second client calling ``claim_channel`` silently + replaces the previous binding; the prior leader stops receiving events. +* **Lossy on leader loss.** If the bound session goes away, the next + ``send_message`` raises and ``ChannelSink`` swallows it as a warning. + Events drop until another client claims. +* **stdio = trivially the one client.** Under stdio there is exactly one + session ever; calling ``claim_channel`` once after init binds it for + the rest of the server's life. Until then, channel events drop. +""" + +from ..base_step import BaseStep +from ...components import R + + +@R.register("claim_channel_step") +class ClaimChannelStep(BaseStep): + """Bind the current MCP session as the ```` recipient.""" + + async def execute(self): + assert self.context is not None + + try: + from fastmcp.server.dependencies import get_context + + ctx = get_context() + session = ctx.session + assert session is not None, "FastMCP context has no ServerSession" + assert self.app_context is not None, "claim_channel requires an application context" + sink = self.app_context.metadata.get("channel_sink") + assert sink is not None, "channel_sink not configured on application context metadata" + except Exception as e: + self.context.response.answer = {"claimed": False, "reason": f"{type(e).__name__}: {e}"} + self.context.response.metadata["claimed"] = False + return self.context.response + + sink.bind(session) + session_id = ctx.session_id or "" + self.logger.info(f"[claim_channel] channel bound to session={session_id}") + self.context.response.answer = { + "claimed": True, + "session_id": session_id, + "note": ( + "this session now receives notifications. " + "last-claim-wins: another call to claim_channel takes over." + ), + } + self.context.response.metadata["claimed"] = True + return self.context.response diff --git a/reme4/steps/index/node_search.py b/reme4/steps/index/node_search.py new file mode 100644 index 00000000..cef24c82 --- /dev/null +++ b/reme4/steps/index/node_search.py @@ -0,0 +1,158 @@ +"""``node_search_step`` — node-level digest search for dream Phase 2. + +Specialized for dream's recall needs; **NOT** a drop-in replacement for the +general-purpose ``search`` step (which serves external RAG agents). + +Five differences vs ``search``: + +1. **Node-level results** — same-path chunks aggregated by max score; one + row per digest node, not per chunk. +2. **Frontmatter included** — returns ``name + description`` inline so the + caller can triage without a follow-up ``frontmatter_read`` per hit. +3. **Digest-only filter** — hardcoded to ``/`` prefix; dream + never wants daily / resource hits as recall candidates. +4. **No expand_links** — dream's synapse recall is looking for nodes that + *don't* yet have wikilinks; expansion would surface already-linked + neighbors (anti-pattern for synapse construction). +5. **No body / chunk text in response** — caller follows up with ``read`` + only on the few candidates that need deep inspection, not all. + +A single hybrid (vector + BM25 RRF) recall serves **both** the dedup +judgment (`same_abstraction` label — is any candidate the same as the +new unit?) and the synapse judgment (`related` label — which candidates +should be woven as wikilinks?). They are two LLM-internal labels over +the **same candidate pool**; there is no need to split into separate +recall passes (the previous ``mode={hybrid,vector_only}`` toggle was +spurious — both judgments operate on the same recall output). + +Used in dream Phase 2 (dream.yaml ``integrate_system_prompt_*``). +External agents must keep using ``search`` for chunk-level retrieval + +link expansion. +""" + +import asyncio +from collections import defaultdict + +from ..base_step import BaseStep +from ...components import R + +_RRF_K = 60 +_MAX_CANDIDATES = 200 + + +def _rrf_merge_nodes( + vector_paths: list[str], + keyword_paths: list[str], + vector_weight: float, +) -> dict[str, float]: + """RRF fuse two ranked path lists into node-level scores.""" + scores: dict[str, float] = defaultdict(float) + text_weight = 1.0 - vector_weight + for rank, path in enumerate(vector_paths, start=1): + scores[path] += vector_weight / (_RRF_K + rank) + for rank, path in enumerate(keyword_paths, start=1): + scores[path] += text_weight / (_RRF_K + rank) + return scores + + +@R.register("node_search_step") +class NodeSearchStep(BaseStep): + """Node-level digest-only hybrid search for dream Phase 2 recall.""" + + async def execute(self): + assert self.context is not None + query: str = (self.context.get("query", "") or "").strip() + limit: int = int(self.context.get("limit") or 20) + vector_weight: float = float(self.kwargs.get("vector_weight", 0.7)) + candidate_multiplier: float = float(self.kwargs.get("candidate_multiplier", 5.0)) + + if not query: + self.context.response.success = False + self.context.response.answer = "Error: query cannot be empty" + return self.context.response + assert limit > 0, f"limit must be positive, got {limit}" + + digest_dir = getattr(self.app_context.app_config, "digest_dir", "digest") or "digest" + digest_prefix = digest_dir.rstrip("/") + "/" + + # Over-fetch — digest filter drops a lot of raw hits. + candidates = min(_MAX_CANDIDATES, max(1, int(limit * candidate_multiplier))) + + vector_chunks, keyword_chunks = await asyncio.gather( + self.file_store.vector_search(query, candidates, {}), + self.file_store.keyword_search(query, candidates, {}), + ) + + def _node_dedup(chunks: list) -> list[str]: + """Keep first occurrence per path; respect digest prefix only. + + Self-exclusion (e.g. UPDATE target) is the LLM's job: frontmatter + inlining lets the agent recognize self from the candidate list. + No need to push it into a mechanical parameter. + """ + seen: set[str] = set() + out: list[str] = [] + for c in chunks: + if c.path in seen: + continue + if not c.path.startswith(digest_prefix): + continue + seen.add(c.path) + out.append(c.path) + return out + + vector_paths = _node_dedup(vector_chunks) + keyword_paths = _node_dedup(keyword_chunks) + path_to_score = _rrf_merge_nodes(vector_paths, keyword_paths, vector_weight) + + ranked = sorted(path_to_score.items(), key=lambda kv: -kv[1])[:limit] + + # Attach frontmatter from in-memory FileNode metadata (no extra IO). + node_paths = [p for p, _ in ranked] + nodes = await self.file_store.get_nodes(node_paths) if node_paths else [] + path_to_fm: dict[str, dict[str, str]] = {} + for n in nodes: + fm = n.front_matter + path_to_fm[n.path] = { + "name": (getattr(fm, "name", "") or "").strip(), + "description": (getattr(fm, "description", "") or "").strip(), + } + + hits: list[dict] = [] + for path, score in ranked: + fm = path_to_fm.get(path, {}) + hits.append( + { + "path": path, + "score": round(float(score), 4), + "name": fm.get("name", ""), + "description": fm.get("description", ""), + }, + ) + + self.logger.info( + f"[{self.name}] query={query!r} candidates={candidates} " + f"vector_hits={len(vector_chunks)} keyword_hits={len(keyword_chunks)} " + f"returned={len(hits)}", + ) + + lines = [ + f"=== node_search query={query!r} hits={len(hits)}/{candidates} ===", + ] + if not hits: + lines.append("(no digest hits)") + else: + for h in hits: + lines.append( + f"[{h['score']:.4f}] {h['path']}\n" f" name: {h['name']}\n" f" description: {h['description']}", + ) + + self.context.response.success = True + self.context.response.answer = "\n".join(lines) + self.context.response.metadata["hits"] = hits + self.context.response.metadata["counts"] = { + "vector_raw": len(vector_chunks), + "keyword_raw": len(keyword_chunks), + "returned": len(hits), + } + return self.context.response diff --git a/reme4/steps/index/scan_changes.py b/reme4/steps/index/scan_changes.py index 3c1ee7d9..2a8c8c6e 100644 --- a/reme4/steps/index/scan_changes.py +++ b/reme4/steps/index/scan_changes.py @@ -1,61 +1,102 @@ -"""One-shot scan: diff watch_paths vs file_store and write changes into context. +"""One-shot scan: diff watch_paths vs an indexed-state source, write changes into context. -Designed to be chained before ``update_index_step`` so that the second step -performs the actual writes and persistence. +Two symmetric variants — pick the one whose state source matches the loop's +write target so the index loop and the dream loop never contend on the +same component: + +* :class:`ScanStoreChangesStep` (``scan_store_changes_step``) — diffs + against ``file_store``; used by ``update_store_index_loop`` (sole + writer of ``file_store``). +* :class:`ScanCatalogChangesStep` (``scan_catalog_changes_step``) — diffs + against ``file_catalog``; used by ``auto_dream_loop`` (sole writer of + ``file_catalog``). + +Both share the same diff vocabulary (``added`` / ``modified`` / ``deleted``) +and write into ``context["changes"]`` in the same shape, so downstream +steps don't care which variant produced the batch. """ from pathlib import Path +from typing import Iterable -from ..base_step import BaseStep +from ..base_step import BaseStep, Ref from ...components import R +from ...components.file_catalog import BaseFileCatalog +from ...enumeration import ComponentEnum +from ...schema import FileNode -@R.register("scan_changes_step") -class ScanChangesStep(BaseStep): - """One-shot scan: compute added/modified/deleted vs file_store and write to context.""" +def _collect_existing( + raw: list[str] | str, + suffixes: list[str], + vault_path: Path, + recursive: bool, +) -> dict[str, float]: + """Walk watch_paths under ``vault_path`` and return ``{abs_path: st_mtime}``.""" + paths = [raw] if isinstance(raw, str) else raw + watch_paths = [vault_path / x for x in paths if (vault_path / x).exists()] + + existing: dict[str, float] = {} + for path in watch_paths: + candidates = [path] if path.is_file() else (path.rglob("*") if recursive else path.iterdir()) + for p in candidates: + if not p.is_file(): + continue + if suffixes and not any(str(p).endswith("." + s.strip(".")) for s in suffixes): + continue + abs_p = p.absolute() + existing[str(abs_p)] = abs_p.stat().st_mtime + return existing + + +def _diff( + existing: dict[str, float], + nodes: Iterable[FileNode], + vault_path: Path, +) -> tuple[list[dict], dict[str, int]]: + """Compute added/modified/deleted vs ``nodes`` and return (changes, counts).""" + indexed: dict[str, float] = { + str(Path(n.path) if Path(n.path).is_absolute() else vault_path / n.path): n.st_mtime for n in nodes + } + + to_delete = list(indexed.keys() - existing.keys()) + to_add = list(existing.keys() - indexed.keys()) + to_modify = [p for p in existing.keys() & indexed.keys() if existing[p] != indexed[p]] + + changes: list[dict] = ( + [{"change": "added", "path": p} for p in to_add] + + [{"change": "modified", "path": p} for p in to_modify] + + [{"change": "deleted", "path": p} for p in to_delete] + ) + counts = {"added": len(to_add), "modified": len(to_modify), "deleted": len(to_delete)} + return changes, counts + + +class _ScanChangesBase(BaseStep): + """Shared scaffolding: collect on-disk state, defer node-loading to the subclass.""" def __init__(self, recursive: bool = True, **kwargs): super().__init__(**kwargs) self.recursive: bool = recursive + async def _load_indexed_nodes(self) -> Iterable[FileNode]: + raise NotImplementedError + async def execute(self): assert self.context is not None - if self.file_store is None: - raise RuntimeError("file_store is not initialized!") - - raw: list[str] = self.context.get("watch_paths", []) - suffixes: list[str] = self.context.get("suffix_filters", ["md"]) vault_path = self.vault_path + raw: list[str] = self.context.get("watch_paths", []) or [] + suffixes: list[str] = self.context.get("suffix_filters", ["md"]) or ["md"] - paths = [raw] if isinstance(raw, str) else raw - watch_paths = [vault_path / x for x in paths if (vault_path / x).exists()] - - existing: dict[str, float] = {} - for path in watch_paths: - candidates = [path] if path.is_file() else (path.rglob("*") if self.recursive else path.iterdir()) - for p in candidates: - if not p.is_file(): - continue - if suffixes and not any(str(p).endswith("." + s.strip(".")) for s in suffixes): - continue - abs_p = p.absolute() - existing[str(abs_p)] = abs_p.stat().st_mtime - - indexed: dict[str, float] = { - str(Path(n.path) if Path(n.path).is_absolute() else vault_path / n.path): n.st_mtime - for n in await self.file_store.get_nodes() - } - - to_delete = list(indexed.keys() - existing.keys()) - to_add = list(existing.keys() - indexed.keys()) - to_modify = [p for p in existing.keys() & indexed.keys() if existing[p] != indexed[p]] - - changes: list[dict] = ( - [{"change": "added", "path": p} for p in to_add] - + [{"change": "modified", "path": p} for p in to_modify] - + [{"change": "deleted", "path": p} for p in to_delete] + existing = _collect_existing( + raw=raw, + suffixes=suffixes, + vault_path=vault_path, + recursive=self.recursive, ) - counts = {"added": len(to_add), "modified": len(to_modify), "deleted": len(to_delete)} + + nodes = await self._load_indexed_nodes() + changes, counts = _diff(existing, nodes, vault_path) self.context["changes"] = changes if changes: @@ -65,3 +106,23 @@ class ScanChangesStep(BaseStep): self.context.response.metadata["counts"] = counts return self.context.response + + +@R.register("scan_store_changes_step") +class ScanStoreChangesStep(_ScanChangesBase): + """Diff vault against ``file_store``; used by the index loop.""" + + async def _load_indexed_nodes(self) -> Iterable[FileNode]: + if self.file_store is None: + raise RuntimeError("file_store is not initialized!") + return await self.file_store.get_nodes() + + +@R.register("scan_catalog_changes_step") +class ScanCatalogChangesStep(_ScanChangesBase): + """Diff vault against ``file_catalog``; used by the dream loop.""" + + file_catalog: BaseFileCatalog = Ref(BaseFileCatalog, ComponentEnum.FILE_CATALOG) + + async def _load_indexed_nodes(self) -> Iterable[FileNode]: + return await self.file_catalog.get_nodes() diff --git a/reme4/steps/index/traverse.py b/reme4/steps/index/traverse.py index ac5f1ed4..f55ae597 100644 --- a/reme4/steps/index/traverse.py +++ b/reme4/steps/index/traverse.py @@ -104,8 +104,26 @@ class TraverseStep(BaseStep): outbound, inbound = await _build_adjacency(self.file_store) results = _bfs(seeds, depth, direction, outbound, inbound) + self.logger.info( + f"[{self.name}] seeds={seeds!r} depth={depth} direction={direction} " + f"nodes={len(outbound) + len(inbound)} edges={len(results)}", + ) + label = seeds[0] if len(seeds) == 1 else f"{len(seeds)} seeds" + if not results: + answer = f"No edges found from {label}" + else: + header = f"Traversed {len(results)} edge(s) from {label}" + lines = [header, ""] + for r in results: + target = r["path"] + if r["anchor"]: + target = f"{target}#{r['anchor']}" + predicate = r["predicate"] or "-" + lines.append(f"[depth={r['depth']}] {r['via']} --{predicate}--> {target}") + answer = "\n".join(lines) + self.context.response.success = True - self.context.response.answer = f"Traversed {len(results)} edge(s) from {label}" + self.context.response.answer = answer self.context.response.metadata.update({"edges": results, "count": len(results)}) return self.context.response diff --git a/reme4/steps/index/watch_changes.py b/reme4/steps/index/watch_changes.py index dfb978a2..5f88401c 100644 --- a/reme4/steps/index/watch_changes.py +++ b/reme4/steps/index/watch_changes.py @@ -1,4 +1,27 @@ -"""Long-running awatch loop: convert raw changes into update_index calls.""" +"""Long-running awatch loop: convert raw changes into dispatch_step calls. + +Two relevant awatch parameters are exposed verbatim: + +* ``step`` (default ``50ms``) — awatch yields when the entire watcher + has gone this long without new changes (and at least one change is + pending). Raise to ``5 minutes``-ish for ``auto_dream_loop`` so + half-written sync output isn't dreamed mid-write; keep at default + for ``update_store_index_loop`` where every fs change should hit + the index promptly. + +* ``debounce`` (default ``2000ms``) — per-batch ceiling, regardless + of whether activity is still arriving. Set ``debounce > step`` so + ``step`` is the operative limit; otherwise the watcher pre-empts + long-quiet-window setups under bursty writes. + +Both are global to the watcher (not per-path). The two reme watchers +have disjoint ``watch_paths`` (digest vs daily/resource), so global +quiet windows are good enough — no per-path bookkeeping needed. + +awatch internally deduplicates same-path same-change tuples within +the yielded batch, so a file ``modified`` ten times during the +quiet window arrives as one ``(modified, path)`` entry. +""" import asyncio @@ -11,13 +34,14 @@ from ...enumeration import ComponentEnum @R.register("watch_changes_step") class WatchChangesStep(BaseStep): - """Watch files and forward each batch of raw changes to a downstream step.""" + """Watch files and forward each yielded batch to a downstream step.""" def __init__( self, recursive: bool = True, force_polling: bool = True, debounce: int = 2000, + step: int = 50, poll_delay_ms: int = 2000, dispatch_step: str = "", **kwargs, @@ -26,6 +50,7 @@ class WatchChangesStep(BaseStep): self.recursive: bool = recursive self.force_polling: bool = force_polling self.debounce: int = debounce + self.step: int = step self.poll_delay_ms: int = poll_delay_ms self.dispatch_step: str = dispatch_step @@ -52,13 +77,17 @@ class WatchChangesStep(BaseStep): if dispatch_step_cls is None: raise RuntimeError(f"Unregistered step '{self.dispatch_step}'") - self.logger.info(f"Watching: {[str(p) for p in valid_paths]}") + self.logger.info( + f"Watching: {[str(p) for p in valid_paths]} step={self.step}ms debounce={self.debounce}ms", + ) + async for raw_changes in awatch( *valid_paths, watch_filter=self._filter, recursive=self.recursive, force_polling=self.force_polling, debounce=self.debounce, + step=self.step, poll_delay_ms=self.poll_delay_ms, stop_event=stop_event, ): diff --git a/reme4/steps/transfer/download.py b/reme4/steps/transfer/download.py index 8dca0732..99db6420 100644 --- a/reme4/steps/transfer/download.py +++ b/reme4/steps/transfer/download.py @@ -50,9 +50,14 @@ class DownloadStep(BaseStep): if "error" in payload: self.context.response.success = False self.context.response.answer = f"Error: {payload['error']}" + self.logger.info(f"[{self.name}] download failed src={src_path} error={payload['error']!r}") else: self.context.response.success = True self.context.response.answer = f"Downloaded {src_path} → {payload['dst_path']} ({payload['size']} bytes)" + self.logger.info( + f"[{self.name}] src={src_path} dst={payload['dst_path']} " + f"size={payload['size']} mime={payload['mime']}", + ) self.context.response.metadata.update(payload) async def _download(self, src_path: str, dst_path: str, overwrite: bool) -> dict: diff --git a/reme4/steps/transfer/ingest.py b/reme4/steps/transfer/ingest.py index f9f36b52..db638fdf 100644 --- a/reme4/steps/transfer/ingest.py +++ b/reme4/steps/transfer/ingest.py @@ -115,6 +115,7 @@ class IngestStep(BaseStep): prepared, prep_error = _prepare_inputs(path, channel, description, metadata_raw) if prep_error: self._fail({"error": prep_error}) + self.logger.info(f"[{self.name}] ingest failed channel={channel!r} error={prep_error!r}") return try: @@ -131,14 +132,19 @@ class IngestStep(BaseStep): ) except _DuplicateIngest as e: self._fail({"error": str(e)}) + self.logger.info(f"[{self.name}] ingest duplicate channel={channel!r} error={str(e)!r}") return except Exception as e: self._fail({"error": f"{type(e).__name__}: {e}"}) + self.logger.info(f"[{self.name}] ingest crashed channel={channel!r} error={type(e).__name__}: {e}") return self.context.response.success = True self.context.response.answer = f"Ingested {outcome['name']} to {outcome['path']}" self.context.response.metadata.update(outcome) + self.logger.info( + f"[{self.name}] channel={channel} date={outcome['date']} name={outcome['name']} path={outcome['path']}", + ) # ------------------------------------------------------------------ diff --git a/reme4/steps/transfer/upload.py b/reme4/steps/transfer/upload.py index cbcffb32..a60fd367 100644 --- a/reme4/steps/transfer/upload.py +++ b/reme4/steps/transfer/upload.py @@ -40,9 +40,15 @@ class UploadStep(BaseStep): if "error" in payload: self.context.response.success = False self.context.response.answer = f"Error: {payload['error']}" + self.logger.info( + f"[{self.name}] upload failed src={src_path} dst={dst_path} error={payload['error']!r}", + ) else: self.context.response.success = True self.context.response.answer = f"Uploaded {src_path} → {dst_path} ({payload['size']} bytes)" + self.logger.info( + f"[{self.name}] src={src_path} dst={dst_path} size={payload['size']} mime={payload['mime']}", + ) self.context.response.metadata.update(payload) async def _upload(self, src_path: str, dst_path: str, overwrite: bool) -> dict: diff --git a/reme4/utils/service_utils.py b/reme4/utils/service_utils.py index 0bc29c78..35a077cd 100644 --- a/reme4/utils/service_utils.py +++ b/reme4/utils/service_utils.py @@ -73,6 +73,7 @@ def precheck_start(svc_config: dict | None) -> bool: """Pre-flight check for `start`: False if reme is up, exits 1 on port conflict.""" host = (svc_config or {}).get("host") or REME_DEFAULT_HOST port = (svc_config or {}).get("port") or REME_DEFAULT_PORT + port = int(port) status = asyncio.run(find_reme(host, port)) if status == "reme": print(f"reme already running at {host}:{port}") diff --git a/reme4/utils/transcript.py b/reme4/utils/transcript.py new file mode 100644 index 00000000..cbc96ebf --- /dev/null +++ b/reme4/utils/transcript.py @@ -0,0 +1,195 @@ +"""Parse Claude Code transcript JSONL into a plain message slice. + +Both routes (``reme-driver`` external, ``reme-service`` internal) accept +``transcript_path`` as the canonical ``sync`` input — Claude Code hooks +hand us the path, not the message list. Centralising the parse here +keeps the two routes producing identical slices for the same transcript. + +The JSONL format is one record per line. The records we care about: + + {"type": "user", "message": {"role": "user", "content": str | list[block]}, ...} + {"type": "assistant", "message": {"role": "assistant", "content": list[block]}, ...} + +Everything else (``ai-title`` / ``mode`` / ``permission-mode`` / +``file-history-snapshot`` / ``attachment`` / ``last-prompt`` / +``queue-operation`` / ``system``) is metadata or harness chatter and is +ignored. + +Content blocks we recognise (shape from the Anthropic message format): + +* ``{"type": "text", "text": str}`` — appended verbatim +* ``{"type": "tool_use", "name": str, "input": ...}`` — rendered as ``[tool ()]`` +* ``{"type": "tool_result", "content": ...}`` — rendered as ``[tool_result ]`` +* ``{"type": "thinking", "thinking": str}`` — dropped (private reasoning) + +User content frequently contains Claude-Code-injected boilerplate that +isn't part of the real conversation: + +* ``...`` — `bash` command warnings prepended to first user turn +* ``...`` — output of slash commands +* ``...`` — slash command label +* ``...`` — slash command description +* ``...`` — periodic harness reminders + +These are filtered out (whole-message drop if the text is only injected +markers, partial strip otherwise) so the synchronizer sees the actual +user/assistant dialogue. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any + +# Whole-message-drop when the user content is only one of these wrappers. +_INJECTED_TAGS = ( + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", +) + +# Heuristic: if the message starts with one of these AND is short / mostly markup, +# drop it. We keep the regex permissive — false negatives (a real message that +# looks like markup) are recoverable downstream; false positives (dropping real +# user text) are silent and worse. +_DROP_IF_STARTS_WITH = tuple(_INJECTED_TAGS) + + +def load_messages_from_transcript( + transcript_path: str | Path, + *, + include_thinking: bool = False, + tool_input_excerpt: int = 200, +) -> list[dict[str, str]]: + """Read a Claude Code transcript JSONL → list of role/content dicts. + + Returns ``[{role, name, content}, ...]`` in source order, where + ``role`` is ``"user"`` or ``"assistant"`` and ``name`` mirrors role + (so the dicts are directly consumable by AgentScope's ``Msg``, which + requires a ``name`` field). Empty list when the file is missing, + empty, or contains no user/assistant turns. + + Parameters + ---------- + transcript_path: + Absolute or relative path to the transcript JSONL file. + include_thinking: + When True, include ``thinking`` blocks (assistant private reasoning). + Default False — the synchronizer wants observable dialogue. + tool_input_excerpt: + Max chars of a ``tool_use`` input JSON to render inline. Default 200. + """ + path = Path(transcript_path) + if not path.is_file(): + return [] + + messages: list[dict[str, str]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + + record_type = record.get("type") + if record_type not in ("user", "assistant"): + continue + + message = record.get("message") or {} + role = message.get("role") + if role not in ("user", "assistant"): + continue + + text = _render_content( + message.get("content", ""), + include_thinking=include_thinking, + tool_input_excerpt=tool_input_excerpt, + ) + if not text: + continue + if _is_injected_only(text): + continue + + messages.append({"role": role, "name": role, "content": text}) + + return messages + + +def _render_content( + content: Any, + *, + include_thinking: bool, + tool_input_excerpt: int, +) -> str: + if isinstance(content, str): + return content.strip() + + if not isinstance(content, list): + return "" + + parts: list[str] = [] + for block in content: + if not isinstance(block, dict): + continue + btype = block.get("type") + + if btype == "text": + t = (block.get("text") or "").strip() + if t: + parts.append(t) + + elif btype == "thinking": + if include_thinking: + t = (block.get("thinking") or "").strip() + if t: + parts.append(f"[thinking]\n{t}") + + elif btype == "tool_use": + name = block.get("name", "?") + try: + inp = json.dumps(block.get("input"), ensure_ascii=False)[:tool_input_excerpt] + except (TypeError, ValueError): + inp = str(block.get("input"))[:tool_input_excerpt] + parts.append(f"[tool {name}({inp})]") + + elif btype == "tool_result": + inner = block.get("content") + if isinstance(inner, list): + excerpt = _render_content( + inner, + include_thinking=False, + tool_input_excerpt=tool_input_excerpt, + ) + else: + excerpt = str(inner or "") + excerpt = excerpt.strip() + if len(excerpt) > tool_input_excerpt: + excerpt = excerpt[:tool_input_excerpt] + "..." + parts.append(f"[tool_result {excerpt}]") + + return "\n".join(p for p in parts if p).strip() + + +def _is_injected_only(text: str) -> bool: + """True if the text is composed entirely of Claude-Code-injected markers + (no genuine user/assistant dialogue around them). + """ + stripped = text.strip() + if not stripped.startswith(_DROP_IF_STARTS_WITH): + return False + # If it starts with an injected tag, peek whether anything substantive + # follows the closing tag. Cheap heuristic: strip all wrapped ... + # blocks and see what's left. + remaining = re.sub(r"<([a-z-]+)>.*?", "", stripped, flags=re.DOTALL) + return len(remaining.strip()) < 16 # arbitrary "essentially empty" threshold diff --git a/tests4/integration/test_dreamer_cli.sh b/tests4/integration/test_dreamer_cli.sh deleted file mode 100755 index 9072dea3..00000000 --- a/tests4/integration/test_dreamer_cli.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# dreamer CLI integration test (option B). -# -# Seeds a rich workspace via _dreamer_fixture.py, starts `reme start` -# bound to that vault, reindexes so Phase 2 recall can hit the pre- -# seeded digest nodes, then calls `reme dream`. -# -# Usage (from anywhere): -# VAULT_PATH=/tmp/reme-dreamer-test bash tests4/integration/test_dreamer_cli.sh -# VAULT_PATH=/tmp/reme-dreamer-test bash tests4/integration/test_dreamer_cli.sh daily/2026-05-28/auth-refactor/notes.md -# -# Defaults: -# VAULT_PATH unset → /tmp/reme-dreamer-test -# Workspace seeded on first run (idempotent). -# -# Required env (from .env or shell): -# LLM_API_KEY, LLM_BASE_URL, LLM_MODEL_NAME -set -euo pipefail - -VAULT="${VAULT_PATH:-/tmp/reme-dreamer-test}" -INTEGRATION_DIR="$(cd "$(dirname "$0")" && pwd)" -REPO="$(cd "$INTEGRATION_DIR/../.." && pwd)" -LOG="/tmp/test_dreamer_cli_server.log" - -# Resolve input from arg, else default to fixture's input path. -DEFAULT_INPUT="$(python -c "import sys; sys.path.insert(0, '$INTEGRATION_DIR'); from _dreamer_fixture import INPUT_PATH; print(INPUT_PATH)")" -INPUT="${1:-$DEFAULT_INPUT}" - -mkdir -p "$VAULT" -echo "--- seeding fixture under $VAULT" -python "$INTEGRATION_DIR/_dreamer_fixture.py" "$VAULT" - -cd "$REPO" - -echo "" -echo "--- starting reme server (log: $LOG)" -reme start "vault_dir=$VAULT" >"$LOG" 2>&1 & -SERVER_PID=$! -trap 'echo "--- stopping reme server (pid $SERVER_PID)"; kill "$SERVER_PID" 2>/dev/null || true; wait "$SERVER_PID" 2>/dev/null || true' EXIT - -echo "--- waiting for server" -for _ in $(seq 1 30); do - if curl -s -o /dev/null http://localhost:8000/docs 2>/dev/null; then - echo "--- server up" - break - fi - sleep 0.5 -done - -echo "" -echo "--- reindexing vault so Phase 2 recall has something to hit" -reme reindex - -echo "" -echo "=== reme dream path=$INPUT ===" -reme dream "path=$INPUT" -echo "" - -echo "=== digest/ tree after dream ===" -if [ -d "$VAULT/digest" ]; then - find "$VAULT/digest" -name "*.md" | sort | while read -r f; do - echo "" - echo "--- ${f#$VAULT/} ---" - cat "$f" - done -else - echo " (no digest/ created)" -fi diff --git a/tests4/integration/test_dreamer_inproc.py b/tests4/integration/test_dreamer_inproc.py index 03cac9a1..a5102efa 100644 --- a/tests4/integration/test_dreamer_inproc.py +++ b/tests4/integration/test_dreamer_inproc.py @@ -11,12 +11,12 @@ wiki}; Phase 2 dispatches to the bucket-specific integrate prompt and writes via the canonical `write` / `edit` tools. Usage (from anywhere): - VAULT_PATH=/tmp/reme-dreamer-test python tests4/integration/test_dreamer_inproc.py - VAULT_PATH=/tmp/reme-dreamer-test python tests4/integration/test_dreamer_inproc.py \\ + VAULT_PATH=tests4/integration/vault python tests4/integration/test_dreamer_inproc.py + VAULT_PATH=tests4/integration/vault python tests4/integration/test_dreamer_inproc.py \\ daily/2026-05-28/auth-refactor/notes.md Defaults: - VAULT_PATH unset → /tmp/reme-dreamer-test + VAULT_PATH unset → tests4/integration/vault Each run wipes `daily/`, `digest/`, and `reme_metadata/` under the vault before reseeding, so the dreamer always starts from the same fixture state. See _dreamer_fixture.py for what gets created and @@ -27,10 +27,13 @@ Required env (from .env or shell): """ import asyncio +import json import os import sys from pathlib import Path +from agentscope.agent import Agent + # Make `reme4` importable regardless of the caller's cwd; and make the # fixture module importable as a top-level name. REPO_ROOT = Path(__file__).resolve().parents[2] @@ -41,7 +44,56 @@ sys.path.insert(0, str(INTEGRATION_DIR)) # pylint: disable=wrong-import-position from _dreamer_fixture import clean_vault, seed_vault, INPUT_PATH # noqa: E402 -VAULT = os.environ.get("VAULT_PATH", "/tmp/reme-dreamer-test") +VAULT = os.environ.get("VAULT_PATH", "tests4/integration/vault") + + +class _AgentMemoryRecorder: + """Monkey-patches Agent.__init__ to capture every agent created inside + the ``with`` block, then dumps each agent's context history to a jsonl + file under ``/agent_logs/`` on dump(). + + Used to inspect the actual ReAct trace of Phase 1 extract + Phase 2 + integrate (per sub-unit) — what tools were called in what order, what + candidates were recalled, what the LLM decided. + """ + + def __init__(self, vault: Path, prefix: str = "dream"): + self.dump_dir = vault / "agent_logs" + self.prefix = prefix + self.agents: list[Agent] = [] + self._orig_init = None + self.dumped_paths: list[Path] = [] + + def __enter__(self): + self._orig_init = Agent.__init__ + agents = self.agents + orig = self._orig_init + + def _capturing_init(agent_self, *args, **kwargs): + orig(agent_self, *args, **kwargs) + agents.append(agent_self) + + Agent.__init__ = _capturing_init + return self + + def __exit__(self, *exc): + Agent.__init__ = self._orig_init + + async def dump(self) -> list[Path]: + """Dump all captured agents' context to /agent_logs/.""" + self.dump_dir.mkdir(parents=True, exist_ok=True) + for stale in self.dump_dir.glob(f"{self.prefix}_*.jsonl"): + stale.unlink() + + for idx, agent in enumerate(self.agents, 1): + messages = agent.state.context + name = getattr(agent, "name", "agent") or "agent" + out_path = self.dump_dir / f"{self.prefix}_{idx:02d}_{name}.jsonl" + with out_path.open("w", encoding="utf-8") as f: + for msg in messages: + f.write(json.dumps(msg.model_dump(), ensure_ascii=False, default=str) + "\n") + self.dumped_paths.append(out_path) + return self.dumped_paths async def main() -> None: @@ -78,7 +130,12 @@ async def main() -> None: await app.run_job("reindex") print(f"\n--- running dream path={rel_input}") - resp = await app.run_job("dream", path=rel_input) + with _AgentMemoryRecorder(vault, prefix="dream") as recorder: + resp = await app.run_job("dream", path=rel_input) + dumped = await recorder.dump() + print(f"\n--- dumped {len(dumped)} agent memory file(s) to {recorder.dump_dir}") + for p in dumped: + print(f" {p.relative_to(vault)}") print("\n=== Response.success ===") print(resp.success) diff --git a/tests4/unit/test_auto_dream.py b/tests4/unit/test_auto_dream.py new file mode 100644 index 00000000..80e8398f --- /dev/null +++ b/tests4/unit/test_auto_dream.py @@ -0,0 +1,360 @@ +"""Tests for AutoDreamStep — daily-tick + file_catalog dedup. + +AutoDreamStep walks ``daily/.md`` + ``daily//**`` and +diffs the result against ``file_catalog`` (same shape as +``scan_catalog_changes_step``): + +* on-disk path missing from catalog → dream +* on-disk mtime != catalog mtime → dream (modified) +* on-disk mtime == catalog mtime → skip (unchanged) +* catalog entry under today's prefix missing → drop from catalog (deleted) + +Successful dreams (and Phase 1 vacuous skips) upsert the current +``st_mtime`` so the next tick re-dreams only what actually changed. +Failures leave the catalog untouched. + +We mock ``dream_one`` (needs an LLM) and inject a fake ``file_catalog`` +recording every get / upsert / delete / dump. +""" + +# pylint: disable=protected-access + +import asyncio +import os +import tempfile +import warnings +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +from reme4.components.file_catalog import BaseFileCatalog +from reme4.components.runtime_context import RuntimeContext +from reme4.schema import FileNode +from reme4.steps import AutoDreamStep +from reme4.steps.evolve.dream import DreamResult + +warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources") + + +class temp_chdir: + """Context manager that temporarily ``chdir``s into a directory and restores cwd on exit.""" + + def __init__(self, path): + self.path = path + self.old = None + + def __enter__(self): + self.old = os.getcwd() + os.chdir(self.path) + return self + + def __exit__(self, *exc): + os.chdir(self.old) + + +def _touch(path: Path, content: str = "x") -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +def _make_step(vault: Path, today: str, existing_nodes: list[FileNode] | None = None) -> AutoDreamStep: + """AutoDreamStep with vault forced, daily_dir='daily', and a fake catalog.""" + fake_catalog = MagicMock(spec=BaseFileCatalog) + fake_catalog.get_nodes = AsyncMock(return_value=list(existing_nodes or [])) + fake_catalog.upsert = AsyncMock() + fake_catalog.delete = AsyncMock() + fake_catalog.dump = AsyncMock() + + class _Fixed(AutoDreamStep): + @property + def vault_path(self): + return vault + + def _vault_dir(self): + return vault + + def _now(self): + import datetime + + return datetime.datetime.fromisoformat(f"{today}T00:00:00") + + step = _Fixed(file_catalog=fake_catalog, persist=True) + cfg = MagicMock() + cfg.daily_dir = "daily" + cfg.resource_dir = "" + step.app_context = MagicMock() + step.app_context.app_config = cfg + return step + + +def test_scans_date_md_and_date_folder(): + """Both ``daily/.md`` and files under ``daily//`` are picked up; + date.md is dreamed first so the day-index leads.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): + vault = Path(tmpdir).resolve() + today = "2026-06-04" + _touch(vault / "daily" / f"{today}.md") + _touch(vault / "daily" / today / "session-a.md") + _touch(vault / "daily" / today / "session-b.md") + step = _make_step(vault, today) + ctx = RuntimeContext() + + seen: list[str] = [] + + async def _fake_dream(rel, _hint): + seen.append(rel) + return DreamResult(used_llm=True, path=rel, summary="ok") + + with patch.object(step, "dream_one", side_effect=_fake_dream): + resp = await step(ctx) + + assert resp.success + assert resp.metadata["files_scanned"] == 3 + assert resp.metadata["files_dreamed"] == 3 + assert seen[0] == f"daily/{today}.md" + assert seen[1:] == [f"daily/{today}/session-a.md", f"daily/{today}/session-b.md"] + print("✓ test_scans_date_md_and_date_folder passed") + + asyncio.run(run()) + + +def test_resource_dir_is_not_scanned(): + """resource// files are NOT picked up by AutoDreamStep anymore.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): + vault = Path(tmpdir).resolve() + today = "2026-06-04" + _touch(vault / "daily" / f"{today}.md") + _touch(vault / "resource" / today / "spec.pdf") + step = _make_step(vault, today) + step.app_context.app_config.resource_dir = "resource" + ctx = RuntimeContext() + + async def _fake_dream(rel, _hint): + return DreamResult(used_llm=True, path=rel, summary="ok") + + with patch.object(step, "dream_one", side_effect=_fake_dream) as dream_mock: + await step(ctx) + + paths = [c.args[0] for c in dream_mock.call_args_list] + assert paths == [f"daily/{today}.md"] + print("✓ test_resource_dir_is_not_scanned passed") + + asyncio.run(run()) + + +def test_unchanged_files_skipped_via_catalog_mtime(): + """A file whose catalog mtime matches on-disk mtime is NOT dreamed.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): + vault = Path(tmpdir).resolve() + today = "2026-06-04" + note = _touch(vault / "daily" / today / "note.md") + mtime = note.stat().st_mtime + existing = [FileNode(path=f"daily/{today}/note.md", st_mtime=mtime)] + step = _make_step(vault, today, existing_nodes=existing) + ctx = RuntimeContext() + + with patch.object(step, "dream_one") as dream_mock: + resp = await step(ctx) + dream_mock.assert_not_called() + + assert resp.success + assert resp.metadata["files_unchanged"] == 1 + assert resp.metadata["files_dreamed"] == 0 + step.file_catalog.upsert.assert_not_awaited() + step.file_catalog.dump.assert_not_awaited() + print("✓ test_unchanged_files_skipped_via_catalog_mtime passed") + + asyncio.run(run()) + + +def test_changed_file_dreamed_and_catalog_updated(): + """Stale catalog mtime → file is dreamed + catalog upserts new mtime.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): + vault = Path(tmpdir).resolve() + today = "2026-06-04" + note = _touch(vault / "daily" / today / "note.md") + mtime = note.stat().st_mtime + stale = mtime - 999.0 + existing = [FileNode(path=f"daily/{today}/note.md", st_mtime=stale)] + step = _make_step(vault, today, existing_nodes=existing) + ctx = RuntimeContext() + + async def _fake_dream(rel, _hint): + return DreamResult(used_llm=True, path=rel, summary="ok") + + with patch.object(step, "dream_one", side_effect=_fake_dream): + resp = await step(ctx) + + assert resp.success + assert resp.metadata["files_dreamed"] == 1 + assert resp.metadata["files_unchanged"] == 0 + step.file_catalog.upsert.assert_awaited_once() + (nodes,), _ = step.file_catalog.upsert.call_args + assert len(nodes) == 1 + assert nodes[0].path == f"daily/{today}/note.md" + assert nodes[0].st_mtime == mtime # post-dream mtime, not stale + step.file_catalog.dump.assert_awaited_once() + print("✓ test_changed_file_dreamed_and_catalog_updated passed") + + asyncio.run(run()) + + +def test_deleted_file_dropped_from_catalog(): + """Catalog entry under today's prefix with no on-disk file → catalog.delete.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): + vault = Path(tmpdir).resolve() + today = "2026-06-04" + # No on-disk files for today; catalog has a stale entry for today. + existing = [FileNode(path=f"daily/{today}/gone.md", st_mtime=123.0)] + step = _make_step(vault, today, existing_nodes=existing) + ctx = RuntimeContext() + + with patch.object(step, "dream_one") as dream_mock: + resp = await step(ctx) + dream_mock.assert_not_called() + + assert resp.success + assert resp.metadata["files_deleted"] == 1 + step.file_catalog.delete.assert_awaited_once_with([f"daily/{today}/gone.md"]) + step.file_catalog.upsert.assert_not_awaited() + step.file_catalog.dump.assert_awaited_once() + print("✓ test_deleted_file_dropped_from_catalog passed") + + asyncio.run(run()) + + +def test_other_days_catalog_entries_untouched(): + """Catalog entries OUTSIDE today's prefix must not be dropped, even if + they don't appear in today's on-disk scan.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): + vault = Path(tmpdir).resolve() + today = "2026-06-04" + yesterday = "2026-06-03" + # Today: nothing on disk. Catalog has yesterday's entry. + existing = [FileNode(path=f"daily/{yesterday}/note.md", st_mtime=99.0)] + step = _make_step(vault, today, existing_nodes=existing) + ctx = RuntimeContext() + + resp = await step(ctx) + + assert resp.success + assert resp.metadata["files_scanned"] == 0 + assert resp.metadata["files_deleted"] == 0 + step.file_catalog.delete.assert_not_awaited() + step.file_catalog.upsert.assert_not_awaited() + print("✓ test_other_days_catalog_entries_untouched passed") + + asyncio.run(run()) + + +def test_failure_does_not_upsert(): + """A dream error must leave the catalog untouched so the next tick retries.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): + vault = Path(tmpdir).resolve() + today = "2026-06-04" + _touch(vault / "daily" / today / "note.md") + step = _make_step(vault, today) + ctx = RuntimeContext() + + async def _fake_dream(rel, _hint): + return DreamResult(used_llm=False, path=rel, error="boom") + + with patch.object(step, "dream_one", side_effect=_fake_dream): + resp = await step(ctx) + + assert not resp.success + assert resp.metadata["files_failed"] == 1 + step.file_catalog.upsert.assert_not_awaited() + step.file_catalog.dump.assert_not_awaited() + print("✓ test_failure_does_not_upsert passed") + + asyncio.run(run()) + + +def test_phase1_empty_still_upserts(): + """Phase 1 skipped (no abstractions) is still success — still catalogued so + the next tick doesn't redo Phase 1 unnecessarily.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): + vault = Path(tmpdir).resolve() + today = "2026-06-04" + note = _touch(vault / "daily" / today / "note.md") + mtime = note.stat().st_mtime + step = _make_step(vault, today) + ctx = RuntimeContext() + + async def _fake_dream(rel, _hint): + return DreamResult(used_llm=True, path=rel, skipped=True, summary="empty") + + with patch.object(step, "dream_one", side_effect=_fake_dream): + resp = await step(ctx) + + assert resp.success + assert resp.metadata["files_skipped"] == 1 + step.file_catalog.upsert.assert_awaited_once() + (nodes,), _ = step.file_catalog.upsert.call_args + assert nodes[0].st_mtime == mtime + print("✓ test_phase1_empty_still_upserts passed") + + asyncio.run(run()) + + +def test_partial_failure_does_not_block_other_files(): + """One file's failure must not stop other files from being dreamed + catalogued.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): + vault = Path(tmpdir).resolve() + today = "2026-06-04" + _touch(vault / "daily" / today / "a.md") + _touch(vault / "daily" / today / "b.md") + step = _make_step(vault, today) + ctx = RuntimeContext() + + async def _fake_dream(rel, _hint): + if rel.endswith("a.md"): + return DreamResult(used_llm=False, path=rel, error="boom") + return DreamResult(used_llm=True, path=rel, summary="ok") + + with patch.object(step, "dream_one", side_effect=_fake_dream): + resp = await step(ctx) + + assert not resp.success + assert resp.metadata["files_dreamed"] == 1 + assert resp.metadata["files_failed"] == 1 + step.file_catalog.upsert.assert_awaited_once() + (nodes,), _ = step.file_catalog.upsert.call_args + assert [n.path for n in nodes] == [f"daily/{today}/b.md"] + print("✓ test_partial_failure_does_not_block_other_files passed") + + asyncio.run(run()) + + +if __name__ == "__main__": + print("\n=== AutoDreamStep Tests ===") + test_scans_date_md_and_date_folder() + test_resource_dir_is_not_scanned() + test_unchanged_files_skipped_via_catalog_mtime() + test_changed_file_dreamed_and_catalog_updated() + test_deleted_file_dropped_from_catalog() + test_other_days_catalog_entries_untouched() + test_failure_does_not_upsert() + test_phase1_empty_still_upserts() + test_partial_failure_does_not_block_other_files() + print("\n所有测试通过!") diff --git a/tests4/unit/test_background_steps.py b/tests4/unit/test_background_steps.py index b3cd50bd..afc83dff 100644 --- a/tests4/unit/test_background_steps.py +++ b/tests4/unit/test_background_steps.py @@ -1,11 +1,15 @@ -"""Tests for background steps: ScanChangesStep + WatchChangesStep. +"""Tests for background steps: ScanStoreChangesStep + WatchChangesStep. Both steps are subclasses of BaseStep. To exercise them without spinning up the full ApplicationContext, we pass real (started) file_store/file_parser via the step's kwargs (so the BaseStep _resolve() machinery returns them). -ScanChangesStep writes its result into ``context["changes"]`` for a downstream +ScanStoreChangesStep writes its result into ``context["changes"]`` for a downstream ``update_index_step`` to consume; tests assert against that key directly. + +The catalog-side sibling (ScanCatalogChangesStep) shares the same diff helper +and is exercised through the dream-loop integration tests; covering it here +would duplicate the file_store-diff assertions without adding signal. """ # pylint: disable=protected-access @@ -21,7 +25,7 @@ from watchfiles import Change from reme4.components.file_parser import ChunkedFileParser from reme4.components.file_store import LocalFileStore from reme4.components.runtime_context import RuntimeContext -from reme4.steps import ScanChangesStep, WatchChangesStep +from reme4.steps import ScanStoreChangesStep, WatchChangesStep warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba") warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources") @@ -51,7 +55,7 @@ def write_file(path: Path, content: str = "x") -> Path: # --------------------------------------------------------------------------- -# ScanChangesStep +# ScanStoreChangesStep # --------------------------------------------------------------------------- @@ -59,12 +63,12 @@ async def _make_scan_step( watch_paths: list[str] | str = "vault", suffix_filters: list[str] | None = None, recursive: bool = True, -) -> tuple[ScanChangesStep, RuntimeContext, LocalFileStore, ChunkedFileParser]: +) -> tuple[ScanStoreChangesStep, RuntimeContext, LocalFileStore, ChunkedFileParser]: fs = LocalFileStore(name="test_store", embedding_store="") parser = ChunkedFileParser() await fs.start() await parser.start() - step = ScanChangesStep( + step = ScanStoreChangesStep( recursive=recursive, file_store=fs, file_parser=parser, @@ -246,7 +250,7 @@ def test_watch_changes_filter_only_passes_md(): if __name__ == "__main__": print("\n=== Background Steps Tests ===") - # ScanChangesStep + # ScanStoreChangesStep test_scan_changes_initial_all_added() test_scan_changes_no_changes_emits_empty_list() test_scan_changes_detects_modify_and_delete() diff --git a/tests4/unit/test_channel_notify.py b/tests4/unit/test_channel_notify.py new file mode 100644 index 00000000..c8a8e689 --- /dev/null +++ b/tests4/unit/test_channel_notify.py @@ -0,0 +1,104 @@ +"""Tests for ``ChannelNotifyStep`` — vault-watcher batch → channel event.""" + +import asyncio +from pathlib import Path + +from reme4.components.application_context import ApplicationContext +from reme4.components.service.mcp_service import ChannelSink +from reme4.components.runtime_context import RuntimeContext +from reme4.steps.index.channel_notify import ChannelNotifyStep + + +class _StubSession: + """Capture ``send_message`` payloads instead of writing them to a transport.""" + + def __init__(self) -> None: + self.sent: list = [] + + async def send_message(self, message) -> None: + """Record the outbound ``SessionMessage`` for later assertions.""" + self.sent.append(message) + + +def _run(coro): + """Drive a coroutine on a fresh event loop (tests don't share one).""" + return asyncio.new_event_loop().run_until_complete(coro) + + +def _ctx(changes: list[dict]) -> RuntimeContext: + """Build a ``RuntimeContext`` pre-populated with the step's ``changes`` input.""" + ctx = RuntimeContext() + ctx["changes"] = changes + return ctx + + +def _app_ctx_with_sink(vault: Path, stub: _StubSession | None) -> tuple[ApplicationContext, ChannelSink | None]: + """Build an ``ApplicationContext`` rooted at ``vault``; attach a sink bound to ``stub`` if given.""" + app_ctx = ApplicationContext(vault_dir=str(vault), app_name="reme-test") + if stub is None: + return app_ctx, None + sink = ChannelSink() + sink.bind(stub) + app_ctx.metadata["channel_sink"] = sink + return app_ctx, sink + + +def test_emits_one_event_per_batch_with_relative_paths(tmp_path): + """A batch of changes → exactly one notification with relative paths and a count meta.""" + vault = tmp_path + (vault / "resource" / "2026-06-03").mkdir(parents=True) + f1 = vault / "resource" / "2026-06-03" / "a.md" + f1.write_text("x") + + stub = _StubSession() + app_ctx, _ = _app_ctx_with_sink(vault, stub) + + step = ChannelNotifyStep(app_context=app_ctx) + _run( + step( + context=_ctx( + [ + {"change": "added", "path": str(f1)}, + {"change": "modified", "path": str(f1)}, + ], + ), + ), + ) + + assert len(stub.sent) == 1 + params = stub.sent[0].message.root.params + assert params["meta"] == {"kind": "vault_change", "count": "2"} + assert "added: resource/2026-06-03/a.md" in params["content"] + assert "modified: resource/2026-06-03/a.md" in params["content"] + + +def test_noop_when_no_changes(tmp_path): + """An empty changes list must not produce any notification.""" + stub = _StubSession() + app_ctx, _ = _app_ctx_with_sink(tmp_path, stub) + step = ChannelNotifyStep(app_context=app_ctx) + _run(step(context=_ctx([]))) + assert not stub.sent + + +def test_noop_when_sink_not_bound(tmp_path): + """Step must run cleanly when no ``ChannelSink`` is configured.""" + app_ctx, _ = _app_ctx_with_sink(tmp_path, None) + step = ChannelNotifyStep(app_context=app_ctx) + # Should run without raising even though channel_sink is absent from metadata + _run(step(context=_ctx([{"change": "added", "path": "/tmp/x.md"}]))) + + +def test_path_outside_vault_passes_through_as_is(tmp_path): + """Paths not under the vault are emitted verbatim instead of crashing.""" + stub = _StubSession() + app_ctx, _ = _app_ctx_with_sink(tmp_path, stub) + step = ChannelNotifyStep(app_context=app_ctx) + _run( + step( + context=_ctx([{"change": "added", "path": "/elsewhere/wild.md"}]), + ), + ) + # Stray absolute path → emitted verbatim, no crash, still one event + assert len(stub.sent) == 1 + assert "added: /elsewhere/wild.md" in stub.sent[0].message.root.params["content"] diff --git a/tests4/unit/test_channel_sink.py b/tests4/unit/test_channel_sink.py new file mode 100644 index 00000000..8b8c89e2 --- /dev/null +++ b/tests4/unit/test_channel_sink.py @@ -0,0 +1,100 @@ +"""Tests for ``ChannelSink`` — outbound ``notifications/claude/channel`` plumbing. + +Strategy: stub a session-like object with an async ``send_message`` capture +list and exercise three paths: + +* not bound → emit is a no-op (no exception, no captured message) +* bound + valid meta → captured JSON-RPC notification carries our method + + content + the meta we passed +* bound + meta with non-identifier keys → the bad keys are dropped, the + rest passes through verbatim +""" + +import asyncio + +from reme4.components.service.mcp_service import ChannelSink + + +class _StubSession: + """Capture ``send_message`` payloads instead of writing them to a transport.""" + + def __init__(self) -> None: + self.sent: list = [] + + async def send_message(self, message) -> None: + """Record the outbound ``SessionMessage`` for later assertions.""" + self.sent.append(message) + + +def _run(coro): + """Drive a coroutine on a fresh event loop (tests don't share one).""" + return asyncio.new_event_loop().run_until_complete(coro) + + +def test_emit_without_bind_is_noop(): + """Emitting before any session is bound must silently no-op.""" + sink = ChannelSink() + _run(sink.emit("hello", {"k": "v"})) # must not raise + + +def test_emit_after_bind_sends_channel_notification(): + """A bound session receives a JSON-RPC notification carrying content + meta verbatim.""" + sink = ChannelSink() + stub = _StubSession() + sink.bind(stub) + + _run(sink.emit("ingest done", {"path": "resource/2026-06-03/x.md", "kind": "ingest"})) + + assert len(stub.sent) == 1 + payload = stub.sent[0].message.root + assert payload.method == "notifications/claude/channel" + assert payload.params["content"] == "ingest done" + assert payload.params["meta"] == {"path": "resource/2026-06-03/x.md", "kind": "ingest"} + + +def test_emit_filters_non_identifier_meta_keys(): + """Meta keys that aren't pure ``[A-Za-z0-9_]`` identifiers are dropped before send.""" + sink = ChannelSink() + stub = _StubSession() + sink.bind(stub) + + _run( + sink.emit( + "x", + { + "good_key": "ok", + "bad-key": "dropped", # hyphen + "also.bad": "dropped", # dot + "Number9": "kept", + }, + ), + ) + + meta = stub.sent[0].message.root.params["meta"] + assert meta == {"good_key": "ok", "Number9": "kept"} + + +def test_unbind_returns_to_noop(): + """After ``unbind``, subsequent emits stop reaching the previously bound session.""" + sink = ChannelSink() + stub = _StubSession() + sink.bind(stub) + sink.unbind() + + _run(sink.emit("x", {})) + assert not stub.sent + + +def test_emit_swallows_send_failures(): + """A failing send_message must not bubble out (notification is best-effort).""" + + class _BoomSession: + """Session whose ``send_message`` always raises, to exercise the failure path.""" + + async def send_message(self, message): + """Raise to simulate a broken transport.""" + raise RuntimeError("transport broke") + + sink = ChannelSink() + sink.bind(_BoomSession()) + _run(sink.emit("x", {})) # must not raise