diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml index 7d63409c..75473b56 100644 --- a/.github/workflows/ci-windows.yml +++ b/.github/workflows/ci-windows.yml @@ -45,7 +45,7 @@ jobs: - name: Run Windows path tests run: | python -m pytest ` - tests/unit/test_auto_dream.py::test_scan_day_files_includes_nested_md_and_excludes_interests ` + tests/unit/test_auto_dream.py::test_scan_day_files_includes_only_markdown_day_files ` tests/unit/test_auto_dream.py::test_dream_extract_matches_posix_catalog_paths ` tests/unit/test_read_with_neighbors.py::test_read_with_neighbors_uses_posix_nested_path ` -v diff --git a/README.md b/README.md index 55485d34..65c6e773 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ cat > .env <<'EOF' # EMBEDDING_API_KEY=sk-xxx # EMBEDDING_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 -# Required for auto_memory, auto_resource, and auto_dream. +# Required for auto_memory, auto_resource, auto_dream, and proactive refresh. LLM_API_KEY=sk-xxx LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 EOF @@ -270,8 +270,8 @@ everything under `metadata/` is rebuildable. | [`auto_memory`](docs/en/auto_memory.md) | Agent hook or `reme auto_memory` | Distills useful conversation facts while preserving a filtered conversation source record. | `session/dialog/*.jsonl`, `daily//.md` | | [`auto_resource`](docs/en/auto_resource.md) | Resource watcher or `reme auto_resource` | Turns files under `resource/` into source-linked, content-named daily cards. | `daily//.md` | | [`auto_index`](docs/en/memory_search.md) | Background watcher or `reme reindex` | The watcher ingests Markdown from `daily/` and `digest/`; `reindex` only rebuilds BM25 and embeddings from already-ingested chunks. | Searchable chunks, BM25, wikilink graph, and optional vectors | -| [`auto_dream`](docs/en/auto_dream.md) | `dream_cron` or `reme auto_dream` | By default, extracts up to five reusable units from changed files in the latest two-day window, then creates, corroborates, refines, or corrects digest nodes. | `digest/**`, `daily//interests.yaml` | -| [`proactive`](docs/en/proactive.md) | `reme proactive` before an agent decides to act | Reads topics generated by `auto_dream`; the host agent decides whether and how to mention them. | Structured topics from `daily//interests.yaml` | +| [`auto_dream`](docs/en/auto_dream.md) | `dream_cron` or `reme auto_dream` | By default, extracts up to five reusable units from changed files in the latest two-day window, then creates, corroborates, refines, or corrects digest nodes. | `digest/**` | +| [`proactive_read`](docs/en/proactive.md) | `reme proactive_read` before an agent decides to act | Reads topics generated by the independent proactive refresh flow; the host agent decides whether and how to mention them. | Structured topics from `daily//interests.yaml` | @@ -297,7 +297,7 @@ BM25 through reciprocal rank fusion (RRF). > [!IMPORTANT] > -> `proactive` only reads and exposes interest topics produced by Auto Dream. It does not independently browse the web, +> `proactive_read` only reads and exposes interest topics produced by proactive refresh. It does not independently browse the web, > send notifications, or rewrite the knowledge base; the host agent decides whether and how to act on a topic. ## 📊 Benchmarks diff --git a/README_ZH.md b/README_ZH.md index 7de559a1..11216b1e 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -98,7 +98,7 @@ cat > .env <<'EOF' # EMBEDDING_API_KEY=sk-xxx # EMBEDDING_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 -# 必须：auto_memory、auto_resource 和 auto_dream 需要 LLM。 +# 必须：auto_memory、auto_resource、auto_dream 和 proactive refresh 需要 LLM。 LLM_API_KEY=sk-xxx LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 EOF @@ -266,8 +266,8 @@ ReMe 遵循 capture → index → consolidate → recall 的循环。workspace | [`auto_memory`](docs/zh/auto_memory.md) | Agent hook 或 `reme auto_memory` | 提炼有长期价值的对话事实，同时保留过滤后的对话来源记录。 | `session/dialog/*.jsonl`、`daily//.md` | | [`auto_resource`](docs/zh/auto_resource.md) | 资源监听或 `reme auto_resource` | 将 `resource/` 下的文件转为带来源链接、按内容命名的 daily 卡片。 | `daily//.md` | | [`auto_index`](docs/zh/memory_search.md) | 后台监听或 `reme reindex` | watcher 摄取 `daily/` 和 `digest/` 中的 Markdown；`reindex` 只基于已摄取的 chunks 重建 BM25 和 Embedding。 | 可检索的 chunks、BM25、wikilink 图谱和可选向量 | -| [`auto_dream`](docs/zh/auto_dream.md) | `dream_cron` 或 `reme auto_dream` | 默认从最近两天内变化的文件中最多提取 5 个可复用 unit，再创建、印证、补充或修正 digest 节点。 | `digest/**`、`daily//interests.yaml` | -| [`proactive`](docs/zh/proactive.md) | Agent 决定主动行动前调用 `reme proactive` | 读取 `auto_dream` 生成的 topics；是否以及如何提醒用户由宿主 Agent 决定。 | 来自 `daily//interests.yaml` 的结构化 topics | +| [`auto_dream`](docs/zh/auto_dream.md) | `dream_cron` 或 `reme auto_dream` | 默认从最近两天内变化的文件中最多提取 5 个可复用 unit，再创建、印证、补充或修正 digest 节点。 | `digest/**` | +| [`proactive_read`](docs/zh/proactive.md) | Agent 决定主动行动前调用 `reme proactive_read` | 读取独立 proactive refresh 流程生成的 topics；是否以及如何提醒用户由宿主 Agent 决定。 | 来自 `daily//interests.yaml` 的结构化 topics |
@@ -292,7 +292,7 @@ ReMe 遵循 capture → index → consolidate → recall 的循环。workspace > [!IMPORTANT] > -> `proactive` 只读取并暴露 Auto Dream 生成的兴趣主题，不会自行联网、发送通知或改写知识库；是否以及如何使用主题，由宿主 Agent +> `proactive_read` 只读取并暴露 proactive refresh 生成的兴趣主题，不会自行联网、发送通知或改写知识库；是否以及如何使用主题，由宿主 Agent > 决定。 ## 📊 评测结果 diff --git a/docs/en/auto_dream.md b/docs/en/auto_dream.md index bb6177f2..1f0a0f5d 100644 --- a/docs/en/auto_dream.md +++ b/docs/en/auto_dream.md @@ -2,16 +2,12 @@ `auto_dream` is ReMe's long-term memory distillation flow from daily to digest. By default it scans the target date and the previous day, processes only files changed since the previous dream, extracts a small set of high-value memory units -across that window, integrates them into `digest/`, and writes the target day's `interests.yaml` for proactive use. - -

- ReMe Auto Dream and Proactive flow from daily to digest to proactive -

+across that window, and integrates them into `digest/`. Its daily inputs usually come from [Auto Memory](./auto_memory.md) and [Auto Resource](./auto_resource.md). For the file semantics of `digest/`, Sources sections, and wikilinks, see [Memory as File](./memory_as_file.md). For the linking -strategy used during Integrate, see [Auto Link](./auto_link.md). To read `interests.yaml`, -use [Proactive](./proactive.md). +strategy used during Integrate, see [Auto Link](./auto_link.md). Proactive discovery is a separate flow; see +[Proactive](./proactive.md). ## Configuration @@ -33,22 +29,12 @@ auto_dream: max_units: type: integer default: 5 - topic_count: - type: integer - default: 3 - topic_diversity_days: - type: integer - default: 7 steps: - backend: dream_extract_step file_catalog: dream - topic_session_id: interests scan_days: 2 max_units: 5 - backend: dream_integrate_step - - backend: dream_topics_step - topic_count: 3 - topic_diversity_days: 7 - backend: dream_finish_step file_catalog: dream ``` @@ -61,8 +47,6 @@ Parameters: | `hint` | Additional guidance from the caller for the Extract and Integrate stages. | | `scan_days` | Recent-date window ending at `date`; defaults to 2 and has a minimum of 1. | | `max_units` | Maximum reusable units extracted in one run; defaults to 5. | -| `topic_count` | Maximum number of topics written to `interests.yaml`. Defaults to 3. | -| `topic_diversity_days` | Number of past days of `interests.yaml` files considered when avoiding duplicate topics. Defaults to 7. | ## Inputs and Outputs @@ -76,8 +60,7 @@ daily/2026-06-20.md daily/2026-06-20/**/*.md ``` -Every `daily//interests.yaml` in the scan window is excluded from extraction so previous proactive output cannot -feed back into the next run. Final topics are written only for the target date. +Only Markdown day indexes and notes are scanned. Proactive state and `interests.yaml` are not Auto Dream inputs. The main outputs are: @@ -86,10 +69,9 @@ The main outputs are: | `digest/procedure/*.md` | Methods, workflows, runbooks, and executable experience. | | `digest/personal/*.md` | User-, team-, and project-related preferences, facts, and long-term context. | | `digest/wiki/*.md` | General knowledge, concepts, observations, and decision precedents. | -| `daily//interests.yaml` | Topics worth proactive attention from the host agent that day. | | `metadata/file_catalog/dream*` | Dream-specific catalog used to detect changes in daily inputs. | -## Four Stages +## Three Stages ### 1. Extract @@ -97,18 +79,15 @@ The main outputs are: 1. Refresh each `daily/.md` in the scan window. 2. Scan those day indexes and `daily//**/*.md`, comparing mtimes with `file_catalog: dream`. -3. Send all changed files together to the LLM and globally extract two structured result types: `units` and `topics`. +3. Send all changed files together to the LLM and globally extract structured memory `units`. `units` are long-term memory units ready to be distilled into digest. Each has `name`, `bucket`, `summary`, and `paths`. A run returns at most `max_units`; extraction merges cross-file evidence for the same abstraction and drops passing mentions, per-file summaries, and weak candidates without reusable value. `bucket` may only be `procedure`, `personal`, or `wiki`; unknown values are routed to `wiki`. -`topics` are proactive-interest candidates for the day. They contain `title`, `reason`, `evidence`, `keywords`, and -`paths` and are filtered again in the Topics stage. - -If there are no changed files, Extract succeeds with no units; Integrate then has no unit work, Topics preserves any -existing target-day topics, and Finish still performs its normal catalog summary. If files changed but no LLM is +If there are no changed files, Extract succeeds with no units; Integrate then has no unit work, and Finish still +performs its normal catalog summary. If files changed but no LLM is configured, Extract fails because extraction requires an LLM. ### 2. Integrate @@ -140,47 +119,17 @@ There are four integration actions: Successfully integrated units are recorded in `integrate_results`. Failed units enter `failed_units`, and their source paths enter `failed_paths`. The Finish stage does not checkpoint failed paths, ensuring that they can be retried later. -### 3. Topics - -`dream_topics_step` turns topic candidates from Extract into the final `daily//interests.yaml` for the day. - -It reads: - -```text -daily//interests.yaml -daily//interests.yaml -``` - -Existing topics from the same day are preserved, while similar topics from the previous `topic_diversity_days` days are -deduplicated. At most three topics are written by default. With an LLM configured, the LLM selects topics that are more -specific, actionable, and non-repetitive. Without an LLM, the step falls back to local normalization and deduplication. - -Example output format. See [Proactive](./proactive.md) for the interface that reads this file: - -```yaml -date: 2026-06-20 -topic_count: 3 -diversity_days: 7 -topics: - - title: Quality regression in the memory retrieval pipeline - reason: The user has recently made repeated changes to search, node_search, and dream integration. - evidence: daily/2026-06-20/session.md - keywords: - - memory search - - auto dream - paths: - - daily/2026-06-20/session.md -``` - -### 4. Finish +### 3. Finish `dream_finish_step` completes the run: 1. Write successfully processed changed paths to `file_catalog: dream`. -2. Also write the target `daily//interests.yaml` and every refreshed day-index page in the scan window to the - catalog. +2. Also write every refreshed day-index page in the scan window to the catalog. 3. Persist the dream catalog if there were upserts or deletions. -4. Return a summary containing counts for scanned, changed, integrated, topics, checkpoints, and related values. +4. Return a summary containing counts for scanned, changed, integrated, checkpoints, and related values. + +Auto Dream neither reads nor writes proactive state or `interests.yaml`. Those files are owned by the proactive refresh +pipeline; see [Proactive](./proactive.md). Failed paths are not checkpointed. The next `auto_dream` run therefore continues to treat them as changed inputs until integration succeeds. @@ -216,7 +165,6 @@ jobs: - backend: dream_extract_step file_catalog: dream - backend: dream_integrate_step - - backend: dream_topics_step - backend: dream_finish_step file_catalog: dream ``` @@ -232,7 +180,6 @@ the workspace-relative wikilink semantics described in [Memory as File](./memory_as_file.md). `auto_dream` does not invent an overview from nothing. Only content that actually appears in daily input and is -extracted as a unit or topic can enter digest or `interests.yaml`. +extracted as a memory unit can enter digest. -The complete flow depends on an LLM for Extract and Integrate. Topics can perform local deduplication without an LLM, -but that does not mean the full dream flow can run offline. +The complete flow depends on an LLM for Extract and Integrate. diff --git a/docs/en/auto_link.md b/docs/en/auto_link.md index dc016e9e..d704b0fa 100644 --- a/docs/en/auto_link.md +++ b/docs/en/auto_link.md @@ -18,7 +18,6 @@ auto_dream: steps: - dream_extract_step - dream_integrate_step # where auto_link actually happens - - dream_topics_step - dream_finish_step ``` diff --git a/docs/en/configuration.md b/docs/en/configuration.md index 49e84bc1..df811cef 100644 --- a/docs/en/configuration.md +++ b/docs/en/configuration.md @@ -86,7 +86,8 @@ components: Built-in registrations include `openai`, `anthropic`, `dashscope`, `deepseek`, `gemini`, `moonshot`, `ollama`, and `xai`. Their detailed model fields follow the corresponding AgentScope wrappers. -File operations, BM25 search, and wikilink traversal do not require an LLM. Evolution workflows such as `auto_memory`, `auto_resource`, and `auto_dream` do. +File operations, BM25 search, wikilink traversal, and `proactive_read` do not require an LLM. Evolution workflows such +as `auto_memory`, `auto_resource`, `auto_dream`, and proactive refresh do. ## Embeddings diff --git a/docs/en/faq.md b/docs/en/faq.md index 6aeb113a..66dd28c7 100644 --- a/docs/en/faq.md +++ b/docs/en/faq.md @@ -7,7 +7,8 @@ description: Quick answers for ReMe installation, services, models, retrieval, f ## Do basic file operations require a model API key? -No. `write`, `read`, `list`, `stat`, BM25 search, and wikilink traversal work without model credentials. `auto_memory`, `auto_resource`, and `auto_dream` require an LLM. +No. `write`, `read`, `list`, `stat`, BM25 search, wikilink traversal, and `proactive_read` work without model +credentials. `auto_memory`, `auto_resource`, `auto_dream`, and proactive refresh require an LLM. ## Why is search still BM25-only after setting an embedding key? diff --git a/docs/en/framework.md b/docs/en/framework.md index d001dcee..a497b403 100644 --- a/docs/en/framework.md +++ b/docs/en/framework.md @@ -421,7 +421,6 @@ jobs: steps: - backend: dream_extract_step - backend: dream_integrate_step - - backend: dream_topics_step - backend: dream_finish_step ``` @@ -433,9 +432,9 @@ The current implementation uses `croniter` to calculate the next trigger time. T ```mermaid flowchart LR Jobs["default.yaml jobs"] --> BG["background
index_update_loop
resource_watch_loop
digest_watch_loop"] - Jobs --> Cron["cron
dream_cron
optimize_index_cron"] + Jobs --> Cron["cron
dream_cron
proactive_refresh_cron
optimize_index_cron"] Jobs --> Stream["stream
chat"] - Jobs --> Base["base
version / help / health_check / status / app_config
search / node_search / traverse / graph_snapshot / reindex
read / load / read_image / write / save / edit / delete / move / list / stat / frontmatter_*
daily_list / daily_reindex / daily_write
auto_memory / auto_memory_cc / auto_resource / auto_dream / proactive"] + Jobs --> Base["base
version / help / health_check / status / app_config
search / node_search / traverse / graph_snapshot / reindex
read / load / read_image / write / save / edit / delete / move / list / stat / frontmatter_*
daily_list / daily_reindex / daily_write
auto_memory / auto_memory_cc / auto_resource / auto_dream / proactive_refresh / proactive_read"] ``` ## 7. Step Model @@ -803,7 +802,6 @@ jobs: - backend: dream_extract_step file_catalog: dream - backend: dream_integrate_step - - backend: dream_topics_step - backend: dream_finish_step file_catalog: dream ``` diff --git a/docs/en/integrations.md b/docs/en/integrations.md index 1e9bddc0..6e2e828a 100644 --- a/docs/en/integrations.md +++ b/docs/en/integrations.md @@ -31,7 +31,8 @@ An empty search result must remain empty; do not present model inference as reca ## MCP -The default HTTP service exposes streamable HTTP MCP at `http://127.0.0.1:2333/mcp`. Common tools include `search`, `read`, `traverse`, `list`, `auto_memory`, and `proactive`. +The default HTTP service exposes streamable HTTP MCP at `http://127.0.0.1:2333/mcp`. Common tools include `search`, +`read`, `traverse`, `list`, `auto_memory`, and `proactive_read`. Use `service.jobs` to expose a read-only subset or keep write tools in a separate configuration. diff --git a/docs/en/memory_as_file.md b/docs/en/memory_as_file.md index 3ca407b9..0393f30b 100644 --- a/docs/en/memory_as_file.md +++ b/docs/en/memory_as_file.md @@ -96,7 +96,7 @@ The corresponding automatic flows are [Auto Memory](./auto_memory.md), [Auto Res │ ├── YYYY-MM-DD.md # index page for the day │ └── YYYY-MM-DD/ │ ├── .md # topic-named conversation or resource card -│ └── interests.yaml # proactive interest topics generated by auto_dream +│ └── interests.yaml # proactive interest topics generated by proactive refresh └── digest/ # deeply processed layer; reusable personal facts, procedures, and knowledge nodes ├── personal/ │ └── .md # user profile, preferences, and durable personal facts diff --git a/docs/en/proactive.md b/docs/en/proactive.md index 19c0b38d..cf876b78 100644 --- a/docs/en/proactive.md +++ b/docs/en/proactive.md @@ -1,7 +1,7 @@ # Proactive -`proactive` is ReMe's interface for reading proactive memory. It does not reanalyze daily notes or call an LLM. It only -reads the current day's interest topics written by `auto_dream`: +`proactive_read` is ReMe's interface for reading proactive memory. It does not reanalyze daily notes or call an LLM. It +reads interest topics written by the independent proactive refresh flow: ```text daily//interests.yaml @@ -10,56 +10,131 @@ daily//interests.yaml A host agent can use it to learn "what is worth proactive attention today," then decide whether to remind the user, ask a follow-up question, recommend a next step, or produce a proactive insight. -`interests.yaml` is generated by the Topics stage of [Auto Dream](./auto_dream.md). `proactive` only reads and exposes -the result. +`interests.yaml` is generated by the proactive refresh pipeline, scheduled by `proactive_refresh_cron` by default. +`proactive_read` only reads and exposes the result; Auto Dream is an independent daily-to-digest flow and does not read +or write proactive state. ## Configuration -The default configuration is in `reme/config/default.yaml`: +The default configuration is in `reme/config/default.yaml`. It defines the same refresh steps twice: a local-only +one-shot job for maintenance and debugging, and the scheduled job that runs every day at 18:00 in the application +timezone: ```yaml -proactive: +proactive_refresh: + backend: base + enable_serve: false + steps: &proactive_refresh_steps + - backend: proactive_extract_step + file_catalog: proactive + scan_days: 2 + carry_forward_days: 14 + max_carry_forward_topics: 20 + llm_timeout_seconds: 300 + max_chars_per_file: 60000 + max_total_chars: 300000 + - backend: proactive_topics_step + known_threshold: 0.85 + known_threshold_calibrated_for: text-embedding-v4@1024 + min_push_confidence: 0.5 + max_topics: 10 + - backend: proactive_plan_step + - backend: proactive_agenda_step + - backend: proactive_finish_step + file_catalog: proactive + +proactive_refresh_cron: + backend: cron + cron: "0 18 * * *" + steps: *proactive_refresh_steps +``` + +The anchor above is only a compact illustration; `default.yaml` spells out both step lists explicitly. The read job is: + +```yaml +proactive_read: backend: base description: "Proactive: read daily//interests.yaml and expose the latest user-interest topics." parameters: - date: - type: string - default: "" - include_content: - type: boolean - default: true + type: object + properties: + date: + type: string + default: "" + include_content: + type: boolean + default: true + horizon_days: + type: integer + default: 1 + min_confidence: + type: number + default: 0.4 steps: - backend: proactive_step + min_confidence: 0.4 ``` Parameters: -| Parameter | Purpose | -|-------------------|-------------------------------------------------------------------------------------------| -| `date` | Date to read in `YYYY-MM-DD` format. When empty, use today in the application's timezone. | -| `include_content` | Whether to return the raw YAML in the answer and metadata. Defaults to `true`. | +| Parameter | Purpose | +|-------------------|------------------------------------------------------------------------------------------------------| +| `date` | Date to read in `YYYY-MM-DD` format. When empty, use today in the application's timezone. | +| `include_content` | Whether to return the raw YAML in the answer and metadata. Defaults to `true`. | +| `horizon_days` | Read one day's exposure file, or use the truth source for a wider evidence horizon. Defaults to `1`. | +| `min_confidence` | Minimum topic confidence to return. Defaults to `0.4`; legacy topics use `0.5`. | + +### Refresh cost, files, and opt-out + +When no daily Markdown note changed, refresh exits before calling an LLM and does not create a new exposure file. With +changed material, extraction normally makes one LLM call and may retry once after an unusable reply. If push candidates +remain, planning makes one additional call; agenda generation makes one more when there are multiple candidates. A +refresh therefore makes at most four LLM calls with the default chain. + +The refresh pipeline maintains the rebuildable `daily/_proactive.yaml` truth source, writes +`daily//interests.yaml`, and advances the independent `proactive` file catalog. Auto Dream does not read or write +any of those proactive artifacts. + +To disable automatic refresh, use an explicit application config that omits the `proactive_refresh_cron` job. Keep the +local-only `proactive_refresh` job if you still want on-demand maintenance. Because it has `enable_serve: false`, it is +not exposed through HTTP or MCP. ## Input Contract -A typical file looks like this: +A current proactive-refresh file looks like this: ```yaml +version: 2 date: 2026-06-20 -topic_count: 3 -diversity_days: 7 +generated_at: 2026-06-20T18:00:00+08:00 +push: true topics: - - title: Quality regression in the memory retrieval pipeline + - id: baa88ad49cb2 + title: Quality regression in the memory retrieval pipeline reason: The user has recently made repeated changes to search, node_search, and dream integration. + kind: follow_up + confidence: 0.86 + first_seen: 2026-06-20 + last_evidence_at: 2026-06-20 evidence: daily/2026-06-20/session.md - keywords: - - memory search - - auto dream paths: - daily/2026-06-20/session.md +agenda: + - topic_id: baa88ad49cb2 + title: Quality regression in the memory retrieval pipeline + scenario_type: resume_task + opener: Review the latest retrieval regression before the next release. + next_action: Compare the failing query against the previous index snapshot. + preconditions: [] + delivery: in_conversation + linked_memory: [] + order_reason: Recent evidence and a concrete next action. +suppressed: [] ``` -Only the `topics` list is parsed into structured results. Every topic requires at least `title` and `reason`; -`evidence`, `keywords`, and `paths` are supporting fields. +Current v2 topics include stable identity, kind, confidence, evidence dates, and source paths. The reader also accepts +legacy v1 files containing `title`, `reason`, `evidence`, `keywords`, and `paths`; missing v2 confidence falls back to +`0.5`. ## Return Value @@ -76,6 +151,14 @@ metadata: | `skipped` | `true` when the file does not exist. | | `error` | Read or parse error. | | `summary` | Short summary. | +| `agenda` | Today's proactive agenda (optional, v2 files only). | + +When today's `interests.yaml` was produced by the proactive refresh chain with an agenda, +the answer also carries an `agenda` field: the ordered agenda items, each with `topic_id`, +`title`, `scenario_type`, `opener` (a natural conversation opener), `next_action` (the +minimal executable step), `preconditions`, `delivery`, `linked_memory` and `order_reason`. +Agenda items whose topic is resolved or below `min_confidence` are filtered out on read; +the field is absent when the file has no agenda. When the file exists and parses successfully, the answer is structured data. For example: @@ -84,13 +167,30 @@ When the file exists and parses successfully, the answer is structured data. For "summary": "Read 1 proactive topic(s) from daily/2026-06-20/interests.yaml", "topics": [ { + "id": "baa88ad49cb2", "title": "Quality regression in the memory retrieval pipeline", "reason": "The user has recently made repeated changes to search, node_search, and dream integration.", + "kind": "follow_up", + "confidence": 0.86, + "first_seen": "2026-06-20", + "last_evidence_at": "2026-06-20", "evidence": "daily/2026-06-20/session.md", - "keywords": ["memory search", "auto dream"], "paths": ["daily/2026-06-20/session.md"] } ], + "agenda": [ + { + "topic_id": "baa88ad49cb2", + "title": "Quality regression in the memory retrieval pipeline", + "scenario_type": "resume_task", + "opener": "Review the latest retrieval regression before the next release.", + "next_action": "Compare the failing query against the previous index snapshot.", + "preconditions": [], + "delivery": "in_conversation", + "linked_memory": [], + "order_reason": "Recent evidence and a concrete next action." + } + ], "content": "date: 2026-06-20\n..." } ``` @@ -104,44 +204,49 @@ A missing file is not an error. The call succeeds with a skipped result: Skipped: interests file not found at daily/2026-06-20/interests.yaml ``` -This lets a host agent treat "there is no dream result for today yet" as a normal empty state. +This lets a host agent treat "there is no proactive refresh result for today yet" as a normal empty state. ## Running Proactive -CLI: +Run one refresh immediately through the normal application lifecycle: ```bash -reme proactive date=2026-06-20 +reme start job=proactive_refresh date=2026-06-20 +``` + +This command may call the configured LLM and may update `_proactive.yaml`, `interests.yaml`, and the proactive catalog. +It does not run Auto Dream. + +Read the generated topics: + +```bash +reme proactive_read date=2026-06-20 ``` Omit the raw YAML content: ```bash -reme proactive date=2026-06-20 include_content=false +reme proactive_read date=2026-06-20 include_content=false ``` ## Relationship to auto_dream -`proactive` is the downstream read step for `auto_dream`: +Proactive refresh and Auto Dream consume daily notes independently: ```text -daily notes - -> auto_dream - -> daily//interests.yaml - -> proactive - -> host agent +daily notes -> auto_dream -> digest +daily notes -> proactive_refresh_cron -> daily//interests.yaml -> proactive_read -> host agent ``` -The responsibilities are divided as follows. For the complete Extract, Integrate, Topics, and Finish flow, see -[Auto Dream](./auto_dream.md): +The proactive responsibilities are divided as follows: -| Module | Responsibility | -|----------------------|--------------------------------------------------------| -| `dream_extract_step` | Extract topic candidates from changed daily inputs. | -| `dream_topics_step` | Deduplicate, select, and write `interests.yaml`. | -| `proactive_step` | Read `interests.yaml` and expose it to the host agent. | +| Module | Responsibility | +|--------------------------|--------------------------------------------------------| +| `proactive_refresh` | Run the refresh pipeline once from the local CLI. | +| `proactive_refresh_cron` | Run the same writer pipeline every day at 18:00. | +| `proactive_step` | Read `interests.yaml` and expose it to the host agent. | -`proactive` does not modify files, update a catalog, or decide whether the user should be interrupted. It only provides +`proactive_read` does not modify files, update a catalog, or decide whether the user should be interrupted. It only provides the day's topic material. The caller's product policy determines whether, when, and in what tone to push it to the user. ## Failure Modes diff --git a/docs/en/quick_start.md b/docs/en/quick_start.md index 2b317245..57e57cce 100644 --- a/docs/en/quick_start.md +++ b/docs/en/quick_start.md @@ -34,7 +34,7 @@ The static build step requires Node.js 22.13 or newer and makes Studio available Installing the `core` extra is recommended. The current code imports the AgentScope wrapper, and self-evolving memory also depends on it. -To use agent workflows such as `auto_memory`, `auto_resource`, and `auto_dream`, configure an LLM: +To use agent workflows such as `auto_memory`, `auto_resource`, `auto_dream`, and proactive refresh, configure an LLM: ```bash cat > .env <<'EOF' @@ -203,7 +203,7 @@ Distill daily notes into long-term digest memory: ```bash reme auto_dream date=2026-06-20 -reme proactive date=2026-06-20 +reme proactive_read date=2026-06-20 ``` These flows require a working LLM. Without an LLM configuration, start with basic capabilities such as `write`, `read`, diff --git a/docs/en/reme_scene.md b/docs/en/reme_scene.md index 15b22953..6ee0b5ed 100644 --- a/docs/en/reme_scene.md +++ b/docs/en/reme_scene.md @@ -13,9 +13,11 @@ Conversations / external resources | +--> auto_dream | distill daily/ into digest/{personal,procedure,wiki}/ - | and write daily//interests.yaml | - +--> search / node_search / read / traverse / proactive + +--> proactive_refresh_cron + | write daily//interests.yaml + | + +--> search / node_search / read / traverse / proactive_read let agents retrieve, associate, read, and inspect interest topics ``` @@ -58,7 +60,7 @@ daily/ ├── glencore-output-update.md ├── drc-cobalt-policy.md ├── high-nickel-cathode-trend.md - └── interests.yaml # generated after auto_dream + └── interests.yaml # generated by proactive refresh ``` The corresponding flow is: @@ -79,17 +81,15 @@ Run: reme auto_dream date=2026-05-18 ``` -`auto_dream` is a four-step pipeline: +`auto_dream` is a three-step pipeline: ```text dream_extract_step scan the daily window from 2026-05-17 through 2026-05-18 by default - output at most 5 units plus topics from changed files + output at most 5 memory units from changed files dream_integrate_step recall existing digest nodes with node_search for each unit decide CREATE / CORROBORATE / REFINE / CORRECT -dream_topics_step - write daily/2026-05-18/interests.yaml dream_finish_step checkpoint successfully processed daily inputs ``` @@ -220,7 +220,7 @@ the CATL interview record on 2026-05-19. ### Proactive: Read the day's interest topics -`auto_dream` writes: +The independent proactive refresh flow writes: ```text daily/2026-05-18/interests.yaml @@ -229,24 +229,41 @@ daily/2026-05-18/interests.yaml Example: ```yaml +version: 2 date: 2026-05-18 -topic_count: 3 -diversity_days: 7 +generated_at: 2026-05-18T18:00:00+08:00 +push: true topics: - - title: Impact of DRC mining-rights policy on cobalt supply + - id: 9c2aa7bd21bf + title: Impact of DRC mining-rights policy on cobalt supply reason: The user repeatedly mentioned KFM and cobalt-price risk today - keywords: [cobalt, DRC, CMOC, KFM] + kind: follow_up + confidence: 0.7 + first_seen: 2026-05-18 + last_evidence_at: 2026-05-18 + evidence: daily/2026-05-18/cobalt-supply-risk.md paths: - daily/2026-05-18/cobalt-supply-risk.md +agenda: + - topic_id: 9c2aa7bd21bf + title: Impact of DRC mining-rights policy on cobalt supply + scenario_type: resume_task + opener: Review the KFM policy update before the next cobalt-supply decision. + next_action: Compare the latest policy note with the existing supply-risk assessment. + preconditions: [] + delivery: in_conversation + linked_memory: [daily/2026-05-18/cobalt-supply-risk.md] + order_reason: Recent evidence and a concrete next step. +suppressed: [] ``` Call: ```bash -reme proactive date=2026-05-18 +reme proactive_read date=2026-05-18 ``` -The `proactive` Job returns the topics from `interests.yaml` and, optionally, the raw YAML content. +The `proactive_read` Job returns the topics from `interests.yaml` and, optionally, the raw YAML content. ### Value of this scenario diff --git a/docs/figure/auto-dream-and-proactive.svg b/docs/figure/auto-dream-and-proactive.svg index 8ebe99cd..48351325 100644 --- a/docs/figure/auto-dream-and-proactive.svg +++ b/docs/figure/auto-dream-and-proactive.svg @@ -1,8 +1,8 @@ ReMe auto dream and proactive flow - A left-to-right flow from a recent changed-daily window to digest integration, interest topic - writing, catalog checkpointing, and proactive reads. + Independent Auto Dream and proactive flows consume daily notes: Auto Dream writes digest memory, + while proactive refresh writes interest topics for proactive reads. @@ -17,7 +17,7 @@ Auto Dream and Proactive - Scan a recent daily window, integrate a compact set of reusable units, then expose proactive topics. + Independent flows turn daily notes into digest memory and proactive topics. @@ -32,7 +32,7 @@ changed daily LLM extract - ≤ 5 units + topics + ≤ 5 memory units @@ -52,37 +52,37 @@ 3 - Topics - dream_topics_step + Finish + dream_finish_step - merge topics - same day kept + checkpoint + skip failures - avoid repeats - last 7 days + persist catalog + dream catalog - write YAML - interests.yaml + return summary + counts + errors - 4 - Finish - dream_finish_step + P + Refresh + proactive_refresh_cron - checkpoint - skip failures + extract topics + changed daily - persist catalog - file_catalog + plan agenda + filter + order - return summary - counts + errors + write YAML + interests.yaml - 5 - Proactive + R + Read proactive_step read YAML @@ -96,7 +96,6 @@ - @@ -112,6 +111,5 @@ Boundary Read-only; caller decides - - proactive reads interests.yaml after auto_dream writes it + Auto Dream and proactive refresh are independent consumers of daily notes diff --git a/docs/zh/auto_dream.md b/docs/zh/auto_dream.md index 6bfc8ea6..d1bda782 100644 --- a/docs/zh/auto_dream.md +++ b/docs/zh/auto_dream.md @@ -1,16 +1,11 @@ # Auto Dream `auto_dream` 是 ReMe 的 daily 到 digest 的长期记忆沉淀流程。它默认扫描目标日期及前一天的 daily 输入，只处理相对上次 dream -发生变化的文件，从整个扫描窗口中抽取少量高价值 memory units，整合进 `digest/`，再生成目标日期可供主动提醒使用的 -`interests.yaml`。 - -

- ReMe Auto Dream and Proactive 从 daily 到 digest 再到 proactive 的流程 -

+发生变化的文件，从整个扫描窗口中抽取少量高价值 memory units，并整合进 `digest/`。 它消费的 daily 输入通常来自 [Auto Memory](./auto_memory.md) 和 [Auto Resource](./auto_resource.md)。`digest/`、Sources 章节 和 wikilink 的文件语义见 [Memory as File](./memory_as_file.md)；Integrate 阶段的链接策略详见 [Auto Link](./auto_link.md)。 -`interests.yaml` 的读取接口见 [Proactive](./proactive.md)。 +主动发现是独立流程，见 [Proactive](./proactive.md)。 ## 配置入口 @@ -32,22 +27,12 @@ auto_dream: max_units: type: integer default: 5 - topic_count: - type: integer - default: 3 - topic_diversity_days: - type: integer - default: 7 steps: - backend: dream_extract_step file_catalog: dream - topic_session_id: interests scan_days: 2 max_units: 5 - backend: dream_integrate_step - - backend: dream_topics_step - topic_count: 3 - topic_diversity_days: 7 - backend: dream_finish_step file_catalog: dream ``` @@ -60,8 +45,6 @@ auto_dream: | `hint` | 调用方给抽取和整合阶段的额外指导。 | | `scan_days` | 以 `date` 结尾的最近日期窗口；默认扫描 2 天，最小为 1。 | | `max_units` | 一次最多抽取多少个可复用 unit；默认 5。 | -| `topic_count` | 最终写入 `interests.yaml` 的 topic 上限，默认 3。 | -| `topic_diversity_days` | 选择 topic 时参考过去多少天的 `interests.yaml` 避免重复，默认 7。 | ## 输入和输出 @@ -74,7 +57,7 @@ daily/2026-06-20.md daily/2026-06-20/**/*.md ``` -扫描窗口内的 `daily//interests.yaml` 都不作为抽取输入，避免上一轮主动主题反过来污染下一轮抽取。最终 topic 只写入目标日期。 +Auto Dream 只扫描 Markdown 日期索引和笔记，不读取 proactive 状态或 `interests.yaml`。 主要输出有三类： @@ -83,10 +66,9 @@ daily/2026-06-20/**/*.md | `digest/procedure/*.md` | 方法、流程、runbook、可执行经验。 | | `digest/personal/*.md` | 用户、团队、项目相关的偏好、事实、长期上下文。 | | `digest/wiki/*.md` | 通用知识、概念、观察、决策先例。 | -| `daily//interests.yaml` | 当天值得上层 Agent 主动关注的兴趣主题。 | | `metadata/file_catalog/dream*` | dream 专用 catalog，用于判断 daily 输入是否变化。 | -## 四个阶段 +## 三个阶段 ### 1. Extract @@ -94,16 +76,14 @@ daily/2026-06-20/**/*.md 1. 刷新扫描窗口内每天的索引页 `daily/.md`。 2. 扫描这些日期的索引页和 `daily//**/*.md`，与 `file_catalog: dream` 中记录的 mtime 对比。 -3. 只把 changed files 一起交给 LLM，全局抽取两类结构化结果：`units` 和 `topics`。 +3. 只把 changed files 一起交给 LLM，全局抽取结构化 memory `units`。 `units` 是准备沉淀进 digest 的长期记忆单元，包含 `name`、`bucket`、`summary`、`paths`。一次最多返回 `max_units` 个，抽取器会优先合并指向同一抽象的跨文件证据，并丢弃短暂提及、逐文件摘要和缺少复用价值的弱候选。`bucket` 只允许 `procedure`、`personal`、`wiki`；未知值会路由到 `wiki`。 -`topics` 是当天主动兴趣候选，包含 `title`、`reason`、`evidence`、`keywords`、`paths`，后续由 Topics 阶段再筛选。 - -如果没有 changed files，Extract 会成功返回空 units；Integrate 随后没有 unit 可处理，Topics 保留目标日期已有的 topics，Finish -仍会正常汇总 catalog。如果有变化但没有配置 LLM，Extract 会失败，因为抽取依赖 LLM。 +如果没有 changed files，Extract 会成功返回空 units；Integrate 随后没有 unit 可处理，Finish 仍会正常汇总 catalog。 +如果有变化但没有配置 LLM，Extract 会失败，因为抽取依赖 LLM。 ### 2. Integrate @@ -131,45 +111,17 @@ digest 节点。新增与更新都必须保留来源，并把相关 digest 链 Integrate 成功的 unit 会记录到 `integrate_results`；失败的 unit 会进入 `failed_units`，其来源路径会进入 `failed_paths`。 Finish 阶段不会 checkpoint 失败路径，保证下次还能重试。 -### 3. Topics - -`dream_topics_step` 将 Extract 阶段产生的 topic candidates 变成当天最终的 `daily//interests.yaml`。 - -它会读取： - -```text -daily//interests.yaml -daily/<过去 topic_diversity_days 天中的每一天>/interests.yaml -``` - -同一天已有 topics 会被保留，最近 `topic_diversity_days` 天出现过的相似主题会被去重。默认最多写 3 个 topic。配置了 LLM 时会让 -LLM 选择更具体、可行动、非重复的主题；没有 LLM 时会退化成本地规范化去重。 - -写入格式示例。读取这个文件的接口见 [Proactive](./proactive.md)： - -```yaml -date: 2026-06-20 -topic_count: 3 -diversity_days: 7 -topics: - - title: 记忆检索链路的质量回归 - reason: 用户近期持续修改 search、node_search 和 dream 集成链路。 - evidence: daily/2026-06-20/session.md - keywords: - - memory search - - auto dream - paths: - - daily/2026-06-20/session.md -``` - -### 4. Finish +### 3. Finish `dream_finish_step` 负责收尾： 1. 将成功处理的 changed paths 写入 `file_catalog: dream`。 -2. 将目标日期的 `daily//interests.yaml` 和扫描窗口内每个已刷新的 day-index 页也写入 catalog。 +2. 将扫描窗口内每个已刷新的 day-index 页也写入 catalog。 3. 如果有 upsert 或 delete，持久化 dream catalog。 -4. 返回包含 scanned、changed、integrated、topics、checkpoint 等计数的摘要。 +4. 返回包含 scanned、changed、integrated、checkpoint 等计数的摘要。 + +Auto Dream 不读取或写入 proactive 状态和 `interests.yaml`。这些文件由 proactive refresh writer 链路负责， +见 [Proactive](./proactive.md)。 失败路径不会被 checkpoint。这样下一次 `auto_dream` 仍会把它们视作 changed input，直到整合成功。 @@ -204,7 +156,6 @@ jobs: - backend: dream_extract_step file_catalog: dream - backend: dream_integrate_step - - backend: dream_topics_step - backend: dream_finish_step file_catalog: dream ``` @@ -217,7 +168,6 @@ jobs: `该决策记录在 [[daily//decision.md]] 中。`链接写法遵循 [Memory as File](./memory_as_file.md) 中的 workspace-relative wikilink 语义。 -`auto_dream` 不凭空生成总览。只有 daily 输入中确实出现、并被抽取为 unit 或 topic 的内容，才会进入 digest 或 -`interests.yaml`。 +`auto_dream` 不凭空生成总览。只有 daily 输入中确实出现、并被抽取为 memory unit 的内容，才会进入 digest。 -完整流程依赖 LLM 完成 Extract 和 Integrate。Topics 可以在没有 LLM 时做本地去重，但这不等于完整 dream 能离线运行。 +完整流程依赖 LLM 完成 Extract 和 Integrate。 diff --git a/docs/zh/auto_link.md b/docs/zh/auto_link.md index 6f87b222..bc59af86 100644 --- a/docs/zh/auto_link.md +++ b/docs/zh/auto_link.md @@ -15,7 +15,6 @@ auto_dream: steps: - dream_extract_step - dream_integrate_step # auto_link 的实际发生位置 - - dream_topics_step - dream_finish_step ``` diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index 6fdeb191..4945d88b 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -93,7 +93,8 @@ components: 可注册的内置 backend 包括 `openai`、`anthropic`、`dashscope`、`deepseek`、`gemini`、`moonshot`、`ollama` 和 `xai`。实际字段由对应 AgentScope model wrapper 决定。 -基础文件操作、BM25 检索、wikilink 遍历不需要 LLM。`auto_memory`、`auto_resource`、`auto_dream` 等演化流程需要可用 LLM。 +基础文件操作、BM25 检索、wikilink 遍历和 `proactive_read` 不需要 LLM。`auto_memory`、`auto_resource`、 +`auto_dream` 和 proactive refresh 等演化流程需要可用 LLM。 ## Embedding 配置 diff --git a/docs/zh/faq.md b/docs/zh/faq.md index 563a93bc..b4adf79e 100644 --- a/docs/zh/faq.md +++ b/docs/zh/faq.md @@ -7,7 +7,8 @@ description: ReMe 安装、服务、模型、检索、文件和插件问题的 ## 基础文件操作需要模型 API Key 吗？ -不需要。`write`、`read`、`list`、`stat`、BM25 搜索和 wikilink 遍历可以在没有模型凭据时运行。`auto_memory`、`auto_resource` 和 `auto_dream` 需要 LLM。 +不需要。`write`、`read`、`list`、`stat`、BM25 搜索、wikilink 遍历和 `proactive_read` 可以在没有模型凭据时 +运行。`auto_memory`、`auto_resource`、`auto_dream` 和 proactive refresh 需要 LLM。 ## 为什么配置了 Embedding Key 仍然只有 BM25？ diff --git a/docs/zh/framework.md b/docs/zh/framework.md index b28613cf..d5cf6e27 100644 --- a/docs/zh/framework.md +++ b/docs/zh/framework.md @@ -405,7 +405,6 @@ jobs: steps: - backend: dream_extract_step - backend: dream_integrate_step - - backend: dream_topics_step - backend: dream_finish_step ``` @@ -416,9 +415,9 @@ jobs: ```mermaid flowchart LR Jobs["default.yaml jobs"] --> BG["background
index_update_loop
resource_watch_loop
digest_watch_loop"] - Jobs --> Cron["cron
dream_cron
optimize_index_cron"] + Jobs --> Cron["cron
dream_cron
proactive_refresh_cron
optimize_index_cron"] Jobs --> Stream["stream
chat"] - Jobs --> Base["base
version / help / health_check / status / app_config
search / node_search / traverse / graph_snapshot / reindex
read / load / read_image / write / save / edit / delete / move / list / stat / frontmatter_*
daily_list / daily_reindex / daily_write
auto_memory / auto_memory_cc / auto_resource / auto_dream / proactive"] + Jobs --> Base["base
version / help / health_check / status / app_config
search / node_search / traverse / graph_snapshot / reindex
read / load / read_image / write / save / edit / delete / move / list / stat / frontmatter_*
daily_list / daily_reindex / daily_write
auto_memory / auto_memory_cc / auto_resource / auto_dream / proactive_refresh / proactive_read"] ``` ## 7. Step 模型 @@ -784,7 +783,6 @@ jobs: - backend: dream_extract_step file_catalog: dream - backend: dream_integrate_step - - backend: dream_topics_step - backend: dream_finish_step file_catalog: dream ``` diff --git a/docs/zh/integrations.md b/docs/zh/integrations.md index ce3dd841..671ce4af 100644 --- a/docs/zh/integrations.md +++ b/docs/zh/integrations.md @@ -40,7 +40,7 @@ ReMe 把记忆能力放在独立服务和用户拥有的 workspace 中。Agent - `traverse` - `list` - `auto_memory` -- `proactive` +- `proactive_read` 根据宿主风险模型，可以用 `service.jobs` 只暴露只读工具，或将写入工具放在单独配置中。 diff --git a/docs/zh/memory_as_file.md b/docs/zh/memory_as_file.md index 0cf36a01..2788c199 100644 --- a/docs/zh/memory_as_file.md +++ b/docs/zh/memory_as_file.md @@ -84,7 +84,7 @@ ReMe 用目录表达记忆组织和记忆分层。原始材料先进入 `resourc │ ├── YYYY-MM-DD.md # 当天索引页 │ └── YYYY-MM-DD/ │ ├── .md # 按主题命名的对话或资源卡片 -│ └── interests.yaml # auto_dream 产出的主动兴趣主题 +│ └── interests.yaml # proactive refresh 产出的主动兴趣主题 └── digest/ # 深加工层；可长期复用的个人事实、流程经验、知识节点 ├── personal/ │ └── .md # 用户画像、偏好、长期个人事实 diff --git a/docs/zh/proactive.md b/docs/zh/proactive.md index a85194f8..c423f09f 100644 --- a/docs/zh/proactive.md +++ b/docs/zh/proactive.md @@ -1,6 +1,7 @@ # Proactive -`proactive` 是 ReMe 的主动记忆读取接口。它不重新分析 daily，也不调用 LLM，只读取 `auto_dream` 写出的当天兴趣主题： +`proactive_read` 是 ReMe 的主动记忆读取接口。它不重新分析 daily，也不调用 LLM，只读取独立 proactive refresh +流程写出的兴趣主题： ```text daily//interests.yaml @@ -8,54 +9,125 @@ daily//interests.yaml 上层 Agent 可以用它获取“今天值得主动关注什么”，再决定是否提醒、追问、推荐下一步或生成主动洞察。 -`interests.yaml` 由 [Auto Dream](./auto_dream.md) 的 Topics 阶段生成；`proactive` 只负责读取和暴露结果。 +`interests.yaml` 由 proactive refresh writer 链路生成，默认通过 `proactive_refresh_cron` 定时执行； +`proactive_read` 只负责读取和暴露结果。Auto Dream 是独立的 daily-to-digest 流程，不读取或写入 proactive 状态。 ## 配置入口 -默认配置在 `reme/config/default.yaml`： +默认配置在 `reme/config/default.yaml`。同一组 refresh steps 有两个入口：用于维护和调试的本地 one-shot job， +以及按应用时区每天 18:00 执行的定时任务： ```yaml -proactive: +proactive_refresh: + backend: base + enable_serve: false + steps: &proactive_refresh_steps + - backend: proactive_extract_step + file_catalog: proactive + scan_days: 2 + carry_forward_days: 14 + max_carry_forward_topics: 20 + llm_timeout_seconds: 300 + max_chars_per_file: 60000 + max_total_chars: 300000 + - backend: proactive_topics_step + known_threshold: 0.85 + known_threshold_calibrated_for: text-embedding-v4@1024 + min_push_confidence: 0.5 + max_topics: 10 + - backend: proactive_plan_step + - backend: proactive_agenda_step + - backend: proactive_finish_step + file_catalog: proactive + +proactive_refresh_cron: + backend: cron + cron: "0 18 * * *" + steps: *proactive_refresh_steps +``` + +上面的 anchor 只是为了紧凑展示；`default.yaml` 中显式写出了两组 steps。读取 job 为： + +```yaml +proactive_read: backend: base description: "Proactive: read daily//interests.yaml and expose the latest user-interest topics." parameters: - date: - type: string - default: "" - include_content: - type: boolean - default: true + type: object + properties: + date: + type: string + default: "" + include_content: + type: boolean + default: true + horizon_days: + type: integer + default: 1 + min_confidence: + type: number + default: 0.4 steps: - backend: proactive_step + min_confidence: 0.4 ``` 参数含义： -| 参数 | 作用 | -|-------------------|-----------------------------------------------------------------| -| `date` | 要读取的日期，格式为 `YYYY-MM-DD`。为空时使用应用时区中的今天。 | -| `include_content` | 是否在 answer 和 metadata 中返回 YAML 原文，默认 `true`。 | +| 参数 | 作用 | +|-------------------|-----------------------------------------------------------------------------| +| `date` | 要读取的日期，格式为 `YYYY-MM-DD`。为空时使用应用时区中的今天。 | +| `include_content` | 是否在 answer 和 metadata 中返回 YAML 原文，默认 `true`。 | +| `horizon_days` | 读取单日曝光文件，或在更宽时间窗下读取 truth source；默认 `1`。 | +| `min_confidence` | 返回 topic 的最低置信度，默认 `0.4`；旧版 topic 使用 `0.5`。 | + +### Refresh 成本、文件与关闭方式 + +没有 daily Markdown 发生变化时，refresh 会在调用 LLM 前结束，也不会生成新的曝光文件。有变化素材时，extract +通常调用一次 LLM；回复不可用时最多重试一次。存在 push candidates 时，plan 会再调用一次；候选数大于一个时， +agenda 会再调用一次。因此默认链路单轮最多调用四次 LLM。 + +Refresh 会维护可重建的 truth source `daily/_proactive.yaml`，写入 `daily//interests.yaml`，并推进独立的 +`proactive` file catalog。Auto Dream 不读取或写入这些 proactive 产物。 + +如需关闭自动 refresh，请使用不包含 `proactive_refresh_cron` job 的显式应用配置。若仍需按需维护，可保留 +本地 one-shot 的 `proactive_refresh`；它设置了 `enable_serve: false`，不会暴露到 HTTP 或 MCP。 ## 输入契约 -典型格式如下： +当前 proactive refresh 生成的文件如下： ```yaml +version: 2 date: 2026-06-20 -topic_count: 3 -diversity_days: 7 +generated_at: 2026-06-20T18:00:00+08:00 +push: true topics: - - title: 记忆检索链路的质量回归 + - id: b2120f3573cb + title: 记忆检索链路的质量回归 reason: 用户近期持续修改 search、node_search 和 dream 集成链路。 + kind: follow_up + confidence: 0.86 + first_seen: 2026-06-20 + last_evidence_at: 2026-06-20 evidence: daily/2026-06-20/session.md - keywords: - - memory search - - auto dream paths: - daily/2026-06-20/session.md +agenda: + - topic_id: b2120f3573cb + title: 记忆检索链路的质量回归 + scenario_type: resume_task + opener: 下次发布前先回顾最近的检索回归。 + next_action: 对比失败查询与上一个索引快照。 + preconditions: [] + delivery: in_conversation + linked_memory: [] + order_reason: 证据较新且下一步明确。 +suppressed: [] ``` -只有 `topics` 列表会被解析成结构化结果。每个 topic 至少需要 `title` 和 `reason`；`evidence`、`keywords`、`paths` 是辅助字段。 +当前 v2 topic 包含稳定 ID、类型、置信度、证据日期和来源路径。读取器仍兼容包含 `title`、`reason`、`evidence`、 +`keywords`、`paths` 的 v1 文件；缺少 v2 置信度时按 `0.5` 处理。 ## 返回结果 @@ -71,6 +143,13 @@ topics: | `skipped` | 文件不存在时为 `true`。 | | `error` | 读取或解析异常。 | | `summary` | 简短摘要。 | +| `agenda` | 当日主动议程（可选，仅 v2 文件携带时返回）。 | + +当当天的 `interests.yaml` 由 proactive refresh 链路生成并携带议程时，answer 会附带 +`agenda` 字段：按顺序排列的当日议程条目，每条包含 `topic_id`、`title`、`scenario_type`、 +`opener`（自然口吻的开场白）、`next_action`（最小可执行动作）、`preconditions`、 +`delivery`、`linked_memory` 与 `order_reason`。读侧会过滤已解决或低于 `min_confidence` +的主题对应的议程条目；文件不含议程时该字段不出现。 文件存在且解析成功时，answer 是结构化数据，例如： @@ -79,13 +158,30 @@ topics: "summary": "Read 1 proactive topic(s) from daily/2026-06-20/interests.yaml", "topics": [ { + "id": "b2120f3573cb", "title": "记忆检索链路的质量回归", "reason": "用户最近反复修改了 search、node_search 和 dream integration。", + "kind": "follow_up", + "confidence": 0.86, + "first_seen": "2026-06-20", + "last_evidence_at": "2026-06-20", "evidence": "daily/2026-06-20/session.md", - "keywords": ["memory search", "auto dream"], "paths": ["daily/2026-06-20/session.md"] } ], + "agenda": [ + { + "topic_id": "b2120f3573cb", + "title": "记忆检索链路的质量回归", + "scenario_type": "resume_task", + "opener": "下次发布前先回顾最近的检索回归。", + "next_action": "对比失败查询与上一个索引快照。", + "preconditions": [], + "delivery": "in_conversation", + "linked_memory": [], + "order_reason": "证据较新且下一步明确。" + } + ], "content": "date: 2026-06-20\n..." } ``` @@ -99,43 +195,49 @@ topics: Skipped: interests file not found at daily/2026-06-20/interests.yaml ``` -这让上层 Agent 可以把“今天还没有 dream 结果”当作正常空状态处理。 +这让上层 Agent 可以把“今天还没有 proactive refresh 结果”当作正常空状态处理。 ## 运行方式 -CLI： +通过正常应用生命周期立即执行一次 refresh： ```bash -reme proactive date=2026-06-20 +reme start job=proactive_refresh date=2026-06-20 +``` + +该命令可能调用配置的 LLM，并更新 `_proactive.yaml`、`interests.yaml` 与 proactive catalog；它不会运行 Auto +Dream。 + +读取生成的 topics： + +```bash +reme proactive_read date=2026-06-20 ``` 不返回 YAML 原文： ```bash -reme proactive date=2026-06-20 include_content=false +reme proactive_read date=2026-06-20 include_content=false ``` ## 与 auto_dream 的关系 -`proactive` 是 `auto_dream` 的下游读取步骤： +Proactive refresh 和 Auto Dream 各自独立消费 daily notes： ```text -daily notes - -> auto_dream - -> daily//interests.yaml - -> proactive - -> upper-level agent +daily notes -> auto_dream -> digest +daily notes -> proactive_refresh_cron -> daily//interests.yaml -> proactive_read -> upper-level agent ``` -职责边界如下。更完整的 Extract、Integrate、Topics、Finish 说明见 [Auto Dream](./auto_dream.md)： +Proactive 职责边界如下： -| 模块 | 职责 | -|----------------------|----------------------------------------------| -| `dream_extract_step` | 从 changed daily 输入抽取 topic candidates。 | -| `dream_topics_step` | 去重、筛选并写入 `interests.yaml`。 | -| `proactive_step` | 读取 `interests.yaml`，暴露给上层 Agent。 | +| 模块 | 职责 | +|--------------------------|----------------------------------------------| +| `proactive_refresh` | 从本地 CLI 单次运行 refresh writer 链路。 | +| `proactive_refresh_cron` | 每天 18:00 运行同一套 writer 链路。 | +| `proactive_step` | 读取 `interests.yaml`，暴露给上层 Agent。 | -`proactive` 不修改任何文件，不更新 catalog，也不负责判断是否应该主动打扰用户。它只提供当天主题材料；是否推送、何时推送、用什么语气推送，应由调用方根据产品策略决定。 +`proactive_read` 不修改任何文件，不更新 catalog，也不负责判断是否应该主动打扰用户。它只提供当天主题材料；是否推送、何时推送、用什么语气推送，应由调用方根据产品策略决定。 ## 失败模式 diff --git a/docs/zh/quick_start.md b/docs/zh/quick_start.md index 3c57f626..e72ad633 100644 --- a/docs/zh/quick_start.md +++ b/docs/zh/quick_start.md @@ -33,7 +33,7 @@ cd .. `core` extra 建议安装：当前代码会导入 AgentScope wrapper，自进化记忆也依赖它。 -如果要使用 `auto_memory`、`auto_resource`、`auto_dream` 这类 Agent 流程，再配置 LLM： +如果要使用 `auto_memory`、`auto_resource`、`auto_dream` 和 proactive refresh 这类 Agent 流程，再配置 LLM： ```bash cat > .env <<'EOF' @@ -197,7 +197,7 @@ reme auto_resource changes='[{"path":"resource/2026-06-20/report.md","change":"a ```bash reme auto_dream date=2026-06-20 -reme proactive date=2026-06-20 +reme proactive_read date=2026-06-20 ``` 这些流程需要可用 LLM；未配置 LLM 时请先使用 `write/read/search` 这类基础能力。 diff --git a/docs/zh/reme_scene.md b/docs/zh/reme_scene.md index 25db4bec..efb8bed5 100644 --- a/docs/zh/reme_scene.md +++ b/docs/zh/reme_scene.md @@ -12,9 +12,11 @@ ReMe 的共同模式是： | +--> auto_dream | 从 daily/ 提炼 digest/{personal,procedure,wiki}/ - | 同时写 daily//interests.yaml | - +--> search / node_search / read / traverse / proactive + +--> proactive_refresh_cron + | 写入 daily//interests.yaml + | + +--> search / node_search / read / traverse / proactive_read 供 Agent 检索、联想、读取兴趣主题 ``` @@ -54,7 +56,7 @@ daily/ ├── glencore-output-update.md ├── drc-cobalt-policy.md ├── high-nickel-cathode-trend.md - └── interests.yaml # auto_dream 后生成 + └── interests.yaml # proactive refresh 生成 ``` 对应链路： @@ -73,20 +75,17 @@ daily/ reme auto_dream date=2026-05-18 ``` -`auto_dream` 是四步管线： +`auto_dream` 是三步管线： ```text dream_extract_step 默认扫描 2026-05-17 至 2026-05-18 的 daily 窗口 - 从 changed 文件输出最多 5 个 units 和 topics + 从 changed 文件输出最多 5 个 memory units dream_integrate_step 每个 unit 用 node_search 召回已有 digest 节点 决定 CREATE / CORROBORATE / REFINE / CORRECT -dream_topics_step - 写 daily/2026-05-18/interests.yaml - dream_finish_step checkpoint 成功处理的 daily 输入 ``` @@ -212,7 +211,7 @@ reme traverse path=digest/wiki/钴.md depth=2 direction=both ### Proactive：读取当天兴趣主题 -`auto_dream` 会写： +独立的 proactive refresh 流程会写： ```text daily/2026-05-18/interests.yaml @@ -221,24 +220,41 @@ daily/2026-05-18/interests.yaml 示例： ```yaml +version: 2 date: 2026-05-18 -topic_count: 3 -diversity_days: 7 +generated_at: 2026-05-18T18:00:00+08:00 +push: true topics: - - title: 刚果(金)矿权政策对钴供给的影响 + - id: f7c355661d51 + title: 刚果(金)矿权政策对钴供给的影响 reason: 用户当天多次提到 KFM 矿和钴价风险 - keywords: [钴, 刚果金, 洛阳钼业, KFM] + kind: follow_up + confidence: 0.7 + first_seen: 2026-05-18 + last_evidence_at: 2026-05-18 + evidence: daily/2026-05-18/cobalt-supply-risk.md paths: - daily/2026-05-18/cobalt-supply-risk.md +agenda: + - topic_id: f7c355661d51 + title: 刚果(金)矿权政策对钴供给的影响 + scenario_type: resume_task + opener: 下次判断钴供给前，先看看 KFM 的最新政策变化。 + next_action: 对比最新政策笔记与已有供给风险判断。 + preconditions: [] + delivery: in_conversation + linked_memory: [daily/2026-05-18/cobalt-supply-risk.md] + order_reason: 证据较新且下一步明确。 +suppressed: [] ``` 调用： ```bash -reme proactive date=2026-05-18 +reme proactive_read date=2026-05-18 ``` -`proactive` Job 返回 `interests.yaml` 中的 topics 和可选 YAML 原文。 +`proactive_read` Job 返回 `interests.yaml` 中的 topics 和可选 YAML 原文。 ### 场景价值 diff --git a/reme/config/benchmark.yaml b/reme/config/benchmark.yaml index 4b38238f..d9d51c3e 100644 --- a/reme/config/benchmark.yaml +++ b/reme/config/benchmark.yaml @@ -42,7 +42,7 @@ jobs: # ── Auto dream (same as default.yaml auto_dream, base mode) ── # auto_dream: # backend: base - # description: "Auto-dream: scan today's day-index and daily notes, globally extract merged units/topics, integrate digest units, write interests.yaml, and persist the dream catalog." + # description: "Auto-dream: scan today's day-index and daily notes, globally extract merged units, integrate digest units, and persist the dream catalog." # parameters: # type: object # properties: @@ -62,24 +62,12 @@ jobs: # type: integer # description: "maximum number of extracted memory units" # default: 5 - # topic_count: - # type: integer - # description: "maximum number of final daily interest topics" - # default: 3 - # topic_diversity_days: - # type: integer - # description: "number of previous interests.yaml days to avoid repeating" - # default: 7 # steps: # - backend: dream_extract_step # file_catalog: dream - # topic_session_id: interests # scan_days: 2 # max_units: 5 # - backend: dream_integrate_step - # - backend: dream_topics_step - # topic_count: 3 - # topic_diversity_days: 7 # - backend: dream_finish_step # file_catalog: dream diff --git a/reme/config/default.yaml b/reme/config/default.yaml index 5daa15af..4b26a2f2 100644 --- a/reme/config/default.yaml +++ b/reme/config/default.yaml @@ -67,13 +67,9 @@ jobs: steps: - backend: dream_extract_step file_catalog: dream - topic_session_id: interests scan_days: 2 max_units: 5 - backend: dream_integrate_step - - backend: dream_topics_step - topic_count: 3 - topic_diversity_days: 7 - backend: dream_finish_step file_catalog: dream @@ -83,9 +79,61 @@ jobs: steps: - backend: optimize_index_step + proactive_refresh: + backend: base + enable_serve: false + description: "Refresh proactive topics once for local maintenance or debugging; not exposed by HTTP/MCP." + parameters: + type: object + properties: + date: + type: string + description: "YYYY-MM-DD to refresh; defaults to today in the application's timezone" + default: "" + steps: + - backend: proactive_extract_step + file_catalog: proactive + scan_days: 2 + carry_forward_days: 14 + max_carry_forward_topics: 20 + llm_timeout_seconds: 300 + max_chars_per_file: 60000 + max_total_chars: 300000 + - backend: proactive_topics_step + known_threshold: 0.85 + known_threshold_calibrated_for: text-embedding-v4@1024 + min_push_confidence: 0.5 + max_topics: 10 + - backend: proactive_plan_step + - backend: proactive_agenda_step + - backend: proactive_finish_step + file_catalog: proactive + + proactive_refresh_cron: + backend: cron + cron: "0 18 * * *" # once a day at 18:00 + steps: + - backend: proactive_extract_step + file_catalog: proactive + scan_days: 2 + carry_forward_days: 14 + max_carry_forward_topics: 20 + llm_timeout_seconds: 300 + max_chars_per_file: 60000 + max_total_chars: 300000 + - backend: proactive_topics_step + known_threshold: 0.85 + known_threshold_calibrated_for: text-embedding-v4@1024 + min_push_confidence: 0.5 + max_topics: 10 + - backend: proactive_plan_step + - backend: proactive_agenda_step + - backend: proactive_finish_step + file_catalog: proactive + auto_dream: backend: base - description: "Auto-dream: scan today's day-index and daily notes, globally extract merged units/topics, integrate digest units, write interests.yaml, and persist the dream catalog." + description: "Auto-dream: scan today's day-index and daily notes, globally extract merged units, integrate digest units, and persist the dream catalog." parameters: type: object properties: @@ -105,24 +153,12 @@ jobs: type: integer description: "maximum number of extracted memory units" default: 5 - topic_count: - type: integer - description: "maximum number of final daily interest topics" - default: 3 - topic_diversity_days: - type: integer - description: "number of previous interests.yaml days to avoid repeating" - default: 7 steps: - backend: dream_extract_step file_catalog: dream - topic_session_id: interests scan_days: 2 max_units: 5 - backend: dream_integrate_step - - backend: dream_topics_step - topic_count: 3 - topic_diversity_days: 7 - backend: dream_finish_step file_catalog: dream @@ -198,7 +234,7 @@ jobs: - auto_image_resource_step - auto_text_resource_step - proactive: + proactive_read: backend: base description: "Proactive: read daily//interests.yaml and expose the latest user-interest topics." parameters: @@ -212,8 +248,17 @@ jobs: type: boolean description: "whether to include the raw YAML content in the response answer and metadata" default: true + horizon_days: + type: integer + description: "merge interests.yaml across this many recent days (1 = single-day legacy behaviour)" + default: 1 + min_confidence: + type: number + description: "minimum topic confidence to return (default 0.4 sits safely below the 0.5 fallback; v1 topics fall back to 0.5)" + default: 0.4 steps: - backend: proactive_step + min_confidence: 0.4 version: backend: base @@ -746,6 +791,7 @@ components: # base_url: ${EMBEDDING_BASE_URL:-https://dashscope.aliyuncs.com/compatible-mode/v1} # parameters: { } # + # embedding_store: # default: # backend: local @@ -824,6 +870,8 @@ components: backend: local dream: backend: local + proactive: + backend: local file_chunker: markdown: diff --git a/reme/schema/__init__.py b/reme/schema/__init__.py index c52e5102..a6b16834 100644 --- a/reme/schema/__init__.py +++ b/reme/schema/__init__.py @@ -4,11 +4,14 @@ from .application_config import ApplicationConfig, ComponentConfig, JobConfig from .dream import ( DreamExtractOutput, DreamState, - DreamTopic, DreamUnit, IntegrateOutcome, +) +from .proactive import ( ProactiveResult, - TopicSelectionOutput, + ProactiveState, + ProactiveStateFile, + ProactiveTopic, ) from .emb_node import EmbNode from .file_chunk import FileChunk @@ -27,7 +30,6 @@ __all__ = [ "ComponentConfig", "DreamExtractOutput", "DreamState", - "DreamTopic", "DreamUnit", "EmbNode", "FileChunk", @@ -40,11 +42,13 @@ __all__ = [ "IntegrateOutcome", "JobConfig", "ProactiveResult", + "ProactiveState", + "ProactiveStateFile", + "ProactiveTopic", "Request", "Response", "StreamChunk", "TokenUsage", - "TopicSelectionOutput", "TraverseGraph", "TraverseGraphEdge", "TraverseGraphNode", diff --git a/reme/schema/dream.py b/reme/schema/dream.py index 15628576..105aa957 100644 --- a/reme/schema/dream.py +++ b/reme/schema/dream.py @@ -6,6 +6,9 @@ from pydantic import BaseModel, Field from ..enumeration import DreamBucketEnum +# ProactiveResult moved to reme.schema.proactive; re-exported for import compatibility. +from .proactive import ProactiveResult # noqa: F401 # pylint: disable=unused-import + class DreamUnit(BaseModel): """One cross-file memory unit emitted by global extract.""" @@ -16,21 +19,10 @@ class DreamUnit(BaseModel): paths: list[str] = Field(default_factory=list, description="Workspace-relative source paths.") -class DreamTopic(BaseModel): - """One topic candidate emitted by global extract.""" - - title: str = Field(description="Specific user-interest topic title.") - reason: str = Field(description="Why this topic may interest the user.") - evidence: str = Field(description="Grounded evidence pointer.") - keywords: list[str] = Field(default_factory=list, description="Keywords for de-duplication.") - paths: list[str] = Field(default_factory=list, description="Workspace-relative source paths.") - - class DreamExtractOutput(BaseModel): """Structured output for ``dream_extract_step``.""" units: list[DreamUnit] = Field(default_factory=list) - topics: list[DreamTopic] = Field(default_factory=list) class IntegrateOutcome(BaseModel): @@ -41,24 +33,6 @@ class IntegrateOutcome(BaseModel): note: str = Field(default="", description="Short summary of what landed.") -class TopicSelectionOutput(BaseModel): - """Structured output for daily topic selection.""" - - topics: list[DreamTopic] = Field(default_factory=list) - - -class ProactiveResult(BaseModel): - """Result of reading daily interest topics.""" - - date: str = "" - path: str = "" - topics: list[dict] = Field(default_factory=list) - content: str = "" - skipped: bool = False - error: str = "" - summary: str = "" - - class DreamState(BaseModel): """Shared state passed across the dream steps.""" @@ -78,7 +52,6 @@ class DreamState(BaseModel): existing: dict[str, float] = Field(default_factory=dict) indexed: dict[str, float] = Field(default_factory=dict) units: list[dict] = Field(default_factory=list) - topics: list[dict] = Field(default_factory=list) extract_summary: str = "" integrate_results: list[dict] = Field(default_factory=list) skipped_units: list[dict] = Field(default_factory=list) @@ -86,14 +59,10 @@ class DreamState(BaseModel): nodes_updated: list[str] = Field(default_factory=list) modified_paths: list[str] = Field( default_factory=list, - description="Durable digest or interests files detected as created or changed during this run.", + description="Durable digest files detected as created or changed during this run.", ) failed_units: list[dict] = Field(default_factory=list) failed_paths: list[str] = Field(default_factory=list) - interests_path: str = "" - interests_paths: list[str] = Field(default_factory=list) - topics_written: int = 0 - topic_error: str = "" checkpoint_paths: list[str] = Field(default_factory=list) warnings: list[str] = Field(default_factory=list) errors: list[str] = Field(default_factory=list) diff --git a/reme/schema/proactive.py b/reme/schema/proactive.py new file mode 100644 index 00000000..6b74c2e0 --- /dev/null +++ b/reme/schema/proactive.py @@ -0,0 +1,134 @@ +"""Proactive refresh schemas. + +Defines the topic model (v2), the chain-shared context state, and the +``daily/_proactive.yaml`` truth-source file model. The LLM reply contract is +validated structurally by ``parse_extract_reply`` instead of a model here. +See ``PROACTIVE_SPEC.md`` sections F1/A2 for the full contracts. +""" + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +TOPIC_KINDS = ("follow_up", "interest_extend") + + +def clamp_confidence(value) -> float: + """Coerce confidence into [0, 1]; any conversion failure falls back to 0.5.""" + try: + return min(1.0, max(0.0, float(value))) + except (TypeError, ValueError): + return 0.5 + + +class ProactiveTopic(BaseModel): + """One proactive topic; every field has a default so v1 files parse seamlessly. + + Fallback rules (A2): invalid ``kind`` -> ``interest_extend``; unparseable + ``confidence`` -> 0.5. Missing ``id``, ``first_seen`` and + ``last_evidence_at`` are context-dependent and therefore resolved by the + loaders, not here. + """ + + id: str = "" + title: str = "" + reason: str = "" + kind: str = "interest_extend" + confidence: float = 0.5 + first_seen: str = "" + last_evidence_at: str = "" + evidence: str = "" + paths: list[str] = Field(default_factory=list) + + @field_validator("kind", mode="before") + @classmethod + def _fallback_kind(cls, value): + text = str(value or "").strip() + return text if text in TOPIC_KINDS else "interest_extend" + + @field_validator("confidence", mode="before") + @classmethod + def _fallback_confidence(cls, value): + return clamp_confidence(value) + + @field_validator("paths", mode="before") + @classmethod + def _clean_str_list(cls, value): + if not isinstance(value, list): + return [] + return [str(item).strip() for item in value if str(item).strip()] + + +class ProactiveState(BaseModel): + """Chain-shared proactive context state (``context['proactive']``). + + Extract fills change-detection/carry-forward/LLM output fields; topics fills the + filtering fields plus ``push_candidates`` (today's pushable topics); plan + expands candidates into ``scenario_cards``; agenda selects the ordered + ``agenda`` and records ``suppressed`` candidates with reasons; finish + records the catalog checkpoint. ``plan_llm_calls`` counts plan+agenda LLM + calls separately from extract's ``llm_calls``. + ``file_skip_reason`` is metadata/log only and never persisted to + interests.yaml (v5 simplification R7). + """ + + date: str = "" + daily_dir: str = "daily" + workspace: str = "" + scan_days: int = 2 + carry_forward_days: int = 14 + changed_paths: list[str] = Field(default_factory=list) + changed_mtimes: dict[str, float] = Field(default_factory=dict) + carry_forward_count: int = 0 + carry_forward_prompt: list[ProactiveTopic] = Field(default_factory=list) + llm_calls: int = 0 + follow_ups: list[dict] = Field(default_factory=list) + extends: list[dict] = Field(default_factory=list) + updates: list[dict] = Field(default_factory=list) + early_exit: str = "" + updates_applied: int = 0 + updates_resolved: int = 0 + candidates_in: int = 0 + candidates: list[dict] = Field(default_factory=list) + dropped_missing: int = 0 + dropped_duplicate: int = 0 + dropped_known: int = 0 + topics_out: list[dict] = Field(default_factory=list) + push_candidates: list[dict] = Field(default_factory=list) + scenario_cards: list[dict] = Field(default_factory=list) + agenda: list[dict] = Field(default_factory=list) + suppressed: list[dict] = Field(default_factory=list) + plan_llm_calls: int = 0 + push: bool = False + file_skip_reason: str = "" + interests_path: str = "" + interests_written: bool = False + checkpoint_paths: list[str] = Field(default_factory=list) + duration_ms: int = 0 + + +class ProactiveStateFile(BaseModel): + """On-disk truth-source ``daily/_proactive.yaml`` (F1.3, v5: 3 sections). + + ``resolved`` tombstones carry ``first_seen`` so a resurrected topic can + keep its original age anchor (F2.4 reopen channel). + """ + + model_config = ConfigDict(extra="ignore") + + version: int = 1 + open_topics: list[ProactiveTopic] = Field(default_factory=list) + resolved: list[dict] = Field(default_factory=list) + + +class ProactiveResult(BaseModel): + """Result of reading daily interest topics (F5).""" + + date: str = "" + path: str = "" + topics: list[dict] = Field(default_factory=list) + content: str = "" + skipped: bool = False + error: str = "" + summary: str = "" + push: bool | None = None + generated_at: str = "" + agenda: list[dict] = Field(default_factory=list) diff --git a/reme/steps/evolve/__init__.py b/reme/steps/evolve/__init__.py index 0e6cec95..c32b3b9b 100644 --- a/reme/steps/evolve/__init__.py +++ b/reme/steps/evolve/__init__.py @@ -1,17 +1,26 @@ """Evolve steps.""" -from ._evolve import now +from ._evolve import now, passthrough_response from .auto_image_resource import AutoImageResourceStep from .auto_memory import AutoMemoryStep from .auto_memory_cc import AutoMemoryCCStep from .auto_resource import AutoResourceStep from .auto_text_resource import AutoTextResourceStep from .compressor import CompressorStep -from .dream import DreamExtractStep, DreamFinishStep, DreamIntegrateStep, DreamTopicsStep, ProactiveStep +from .dream import DreamExtractStep, DreamFinishStep, DreamIntegrateStep +from .proactive import ( + ProactiveAgendaStep, + ProactiveExtractStep, + ProactiveFinishStep, + ProactivePlanStep, + ProactiveStep, + ProactiveTopicsStep, +) __all__ = [ "now", "AutoImageResourceStep", + "passthrough_response", "AutoMemoryStep", "AutoMemoryCCStep", "AutoResourceStep", @@ -20,6 +29,10 @@ __all__ = [ "DreamExtractStep", "DreamFinishStep", "DreamIntegrateStep", - "DreamTopicsStep", + "ProactiveAgendaStep", + "ProactiveExtractStep", + "ProactiveFinishStep", + "ProactivePlanStep", "ProactiveStep", + "ProactiveTopicsStep", ] diff --git a/reme/steps/evolve/_evolve.py b/reme/steps/evolve/_evolve.py index 6a2e5514..73a0148b 100644 --- a/reme/steps/evolve/_evolve.py +++ b/reme/steps/evolve/_evolve.py @@ -5,6 +5,8 @@ import zoneinfo from agentscope.message import Msg +from ...schema import Response + def now(timezone: str | None = None) -> datetime.datetime: """Return current datetime in the given IANA timezone, falling back to local.""" @@ -40,3 +42,22 @@ def agent_reply_result_text(reply_result: dict) -> str: if text: return text return str(reply_result.get("result") or "").strip() + + +def passthrough_response(step, skip_key: str) -> Response: + """Return a success response when a short-circuit flag is set (INV-7). + + Short-circuited rounds never write interests.yaml or checkpoint catalogs; + the job still reports success so the skipped round is not counted as a failure. + """ + assert step.context is not None + response = step.context.response + response.success = True + flag = step.context.get(skip_key) + if isinstance(flag, dict): + reason = str(flag.get("reason") or "skipped") + else: + reason = str(flag or "skipped") + response.answer = f"Skipped: {reason}" + step.logger.info(f"[{step.name}] short-circuit via {skip_key!r} reason={reason}") + return response diff --git a/reme/steps/evolve/dream/__init__.py b/reme/steps/evolve/dream/__init__.py index fa6291a7..3766cb01 100644 --- a/reme/steps/evolve/dream/__init__.py +++ b/reme/steps/evolve/dream/__init__.py @@ -3,13 +3,9 @@ from .extract import DreamExtractStep from .finish import DreamFinishStep from .integrate import DreamIntegrateStep -from .proactive import ProactiveStep -from .topics import DreamTopicsStep __all__ = [ "DreamExtractStep", "DreamFinishStep", "DreamIntegrateStep", - "DreamTopicsStep", - "ProactiveStep", ] diff --git a/reme/steps/evolve/dream/extract.py b/reme/steps/evolve/dream/extract.py index b4884de5..59f6b9c9 100644 --- a/reme/steps/evolve/dream/extract.py +++ b/reme/steps/evolve/dream/extract.py @@ -26,11 +26,10 @@ _TOOLS = ("read",) @R.register("dream_extract_step") class DreamExtractStep(BaseStep): - """Scan changed daily files and globally extract merged units/topics.""" + """Scan changed daily files and globally extract merged memory units.""" - def __init__(self, topic_session_id: str = "interests", scan_days: int = 2, max_units: int = 5, **kwargs): + def __init__(self, scan_days: int = 2, max_units: int = 5, **kwargs): super().__init__(**kwargs) - self.topic_session_id = topic_session_id self.scan_days = scan_days self.max_units = max_units @@ -57,22 +56,27 @@ class DreamExtractStep(BaseStep): existing = self._existing( workspace, - [ - path - for scan_day in dates - for path in scan_day_files(workspace, scan_day, daily, f"{self.topic_session_id}.yaml") - ], + [path for scan_day in dates for path in scan_day_files(workspace, scan_day, daily)], ) - interest_rels = {f"{daily}/{scan_day}/{self.topic_session_id}.yaml" for scan_day in dates} day_mds = {f"{daily}/{scan_day}.md" for scan_day in dates} day_prefixes = tuple(f"{daily}/{scan_day}/" for scan_day in dates) nodes = await self.file_catalog.get_nodes() - indexed_all = {n.path: n.st_mtime for n in nodes if n.path in day_mds or n.path.startswith(day_prefixes)} - indexed = {path: mt for path, mt in indexed_all.items() if path not in interest_rels} + # Older Auto Dream versions checkpointed generated interests files. + # Remove every such watermark from the dream catalog, not only entries + # inside the current scan window. The exposure files themselves remain + # untouched and are owned by the proactive refresh pipeline. + legacy_interests = sorted( + {n.path for n in nodes if n.path.startswith(f"{daily}/") and n.path.endswith("/interests.yaml")}, + ) + indexed_all = { + n.path: n.st_mtime + for n in nodes + if n.path not in legacy_interests and (n.path in day_mds or n.path.startswith(day_prefixes)) + } + indexed = {path: mt for path, mt in indexed_all.items() if path in existing} changed = [rel for rel, mt in existing.items() if indexed.get(rel) != mt] unchanged = [rel for rel, mt in existing.items() if indexed.get(rel) == mt] - protected = set(existing) | {rel for rel in interest_rels if (workspace / rel).is_file()} - deleted = sorted(indexed_all.keys() - protected) + deleted = sorted((indexed_all.keys() - set(existing)) | set(legacy_interests)) self.logger.info( f"[{self.name}] scan summary existing={len(existing)} indexed={len(indexed)} " f"changed={len(changed)} unchanged={len(unchanged)} deleted={len(deleted)}", @@ -147,21 +151,20 @@ class DreamExtractStep(BaseStep): return self._finish(state, False, error) units = meta.get("units") if "units" in meta else meta.get("memory_units") - if isinstance(units, list) and isinstance(meta.get("topics"), list): + if isinstance(units, list): break if attempt == 0: self.logger.warning(f"[{self.name}] extract attempt 1 returned an unusable receipt; retrying once") continue # Keep the warning-only result checkpointable after one retry so a bad source cannot loop forever. - warning = "dream extract skipped unusable agent receipt after retry; expected units and topics lists" + warning = "dream extract skipped unusable agent receipt after retry; expected a units list" state.warnings.append(warning) self.logger.warning(f"[{self.name}] {warning}") self.logger.info(f"[{self.name}] parse done keys={','.join(sorted(meta.keys())) if meta else '(none)'}") self.clean_output(state, meta, max_units=max_units) state.extract_summary = raw_result - answer = f"Extracted {len(state.units)} unit(s), {len(state.topics)} topic(s)" - answer = f"{answer} from {len(changed)} changed file(s) across {len(dates)} day(s)" + answer = f"Extracted {len(state.units)} unit(s) from {len(changed)} changed file(s) across {len(dates)} day(s)" return self._finish(state, True, answer) def _existing(self, workspace, files: list[str]) -> dict[str, float]: @@ -193,29 +196,6 @@ class DreamExtractStep(BaseStep): self.logger.warning(f"[{self.name}] unit {name!r} emitted bucket {raw_bucket!r}; routing to wiki") bucket = DreamBucketEnum.WIKI.value state.units.append({"name": name, "bucket": bucket, "summary": summary, "paths": paths}) - for raw in meta.get("topics") or []: - topic = self._clean_topic(raw, allowed) - if topic: - state.topics.append(topic) - - @staticmethod - def _clean_topic(raw, allowed: set[str]) -> dict: - if not isinstance(raw, dict): - return {} - title = str(raw.get("title") or "").strip() - reason = str(raw.get("reason") or "").strip() - paths = clean_paths(raw.get("paths"), allowed) - if not title or not reason or not paths: - return {} - keywords = raw.get("keywords") or [] - cleaned_keywords = [str(k).strip() for k in keywords if str(k).strip()] if isinstance(keywords, list) else [] - return { - "title": title, - "reason": reason, - "evidence": str(raw.get("evidence") or "").strip(), - "keywords": cleaned_keywords, - "paths": paths, - } def _finish(self, state: DreamState, success: bool, answer: str): assert self.context is not None diff --git a/reme/steps/evolve/dream/extract.yaml b/reme/steps/evolve/dream/extract.yaml index 3af85816..b35847fe 100644 --- a/reme/steps/evolve/dream/extract.yaml +++ b/reme/steps/evolve/dream/extract.yaml @@ -1,7 +1,6 @@ extract_system_prompt: | You are the dream global extraction agent. Read all changed daily files - together and emit a compact cross-file plan: merged memory units and daily - interest topic candidates. + together and emit a compact cross-file plan of merged memory units. workspace_dir: {workspace_dir} buckets: {buckets} @@ -14,7 +13,6 @@ extract_system_prompt: | ## What to Extract - **Reusable memory units**: durable abstractions worth integrating into digest. - - **Daily interest candidates**: topics the user may care about seeing again. - **Cross-file merges**: one unit may gather evidence from several paths. - **No raw summaries**: do not summarize every file or every event. @@ -42,7 +40,7 @@ extract_system_prompt: | - summary should name the abstraction, explain why it matters, and point at the supporting evidence; do not quote or summarize a note. - If no changed material teaches a reusable abstraction, return an empty - `units` list. Topic candidates may still be non-empty. + `units` list. ## Bucket Rules @@ -57,13 +55,6 @@ extract_system_prompt: | - "Small PRs are easier to review" -> wiki. - "Steps to split a large PR" -> procedure. - ## Topic Rules - - - Emit topics the user may care about, not generic labels. - - Each topic must include title, reason, evidence, keywords, and paths. - - paths must only contain values from changed_paths. - - Prefer concrete, recurring, or actionable interests over broad categories. - ## Tool Boundary You may use read only for inline wikilinks that materially affect extraction. @@ -76,16 +67,9 @@ extract_system_prompt: | bucket: procedure|personal|wiki summary: paths: [, ...] - topics: - - title: - reason: - evidence: - keywords: [, ...] - paths: [, ...] extract_system_prompt_zh: | - 你是 dream 全局抽取 agent。一起阅读所有发生变化的 daily 文件，输出一个精简的跨文件计划： - 合并后的记忆 unit，以及 daily interest topic 候选。 + 你是 dream 全局抽取 agent。一起阅读所有发生变化的 daily 文件，输出一个由合并记忆 unit 组成的精简跨文件计划。 workspace_dir: {workspace_dir} buckets: {buckets} @@ -97,7 +81,6 @@ extract_system_prompt_zh: | ## 抽取什么 - **可复用记忆 unit**：值得整合进 digest 的长期抽象。 - - **Daily interest 候选**：用户之后可能还会关心、值得保留的主题。 - **跨文件合并**：一个 unit 可以合并来自多个 path 的证据。 - **不要原文摘要**：不要逐文件总结，也不要逐事件总结。 @@ -116,7 +99,7 @@ extract_system_prompt_zh: | - 不要输出 passing mention、已知概念复述、事件 umbrella unit、一次性时间戳、参会事实、 或没有复用价值的事实。 - summary 应命名抽象、解释它为什么重要，并指向支持证据；不要只是摘抄或总结笔记。 - - 如果 changed material 没有教导任何可复用抽象，返回空 `units` list；topic 候选仍然可以非空。 + - 如果 changed material 没有教导任何可复用抽象，返回空 `units` list。 ## Bucket 规则 @@ -129,13 +112,6 @@ extract_system_prompt_zh: | - “小 PR 更容易 review” -> wiki。 - “如何拆分大 PR 的步骤” -> procedure。 - ## Topic 规则 - - - 输出用户可能真正关心的 topic，而不是泛泛的标签。 - - 每个 topic 必须包含 title、reason、evidence、keywords、paths。 - - paths 只能使用 changed_paths 中出现的值。 - - 优先选择具体、反复出现、可行动的兴趣，而不是宽泛类别。 - ## 工具边界 只有当 inline wikilink 会实质影响抽取时，才可以使用 read。 @@ -148,12 +124,6 @@ extract_system_prompt_zh: | bucket: procedure|personal|wiki summary: <基于证据的可复用抽象> paths: [, ...] - topics: - - title: <具体的用户兴趣 topic> - reason: <为什么重要> - evidence: <简短证据指针> - keywords: [<关键词>, ...] - paths: [, ...] extract_user_message: | date: {date} @@ -169,7 +139,7 @@ extract_user_message: | {material_blob} - Extract merged memory units and daily interest topic candidates. + Extract merged memory units. extract_user_message_zh: | 日期：{date} @@ -185,4 +155,4 @@ extract_user_message_zh: | {material_blob} - 抽取合并后的记忆 units 和 daily interest topic 候选。 + 抽取合并后的记忆 units。 diff --git a/reme/steps/evolve/dream/finish.py b/reme/steps/evolve/dream/finish.py index 7ea79995..d1245d82 100644 --- a/reme/steps/evolve/dream/finish.py +++ b/reme/steps/evolve/dream/finish.py @@ -26,12 +26,11 @@ class DreamFinishStep(BaseStep): failed_paths = set(state.failed_paths) checkpoint = [p for p in state.changed_paths if p not in failed_paths] day_index_paths = [f"{state.daily_dir}/{day}.md" for day in (state.dates or [state.date]) if day] - interest_paths = state.interests_paths or ([state.interests_path] if state.interests_path else []) - supplemental = [p for p in [*interest_paths, *day_index_paths] if p and p not in failed_paths] + supplemental = [p for p in day_index_paths if p and p not in failed_paths] upsert_paths = list(dict.fromkeys([*checkpoint, *supplemental])) self.logger.info( f"[{self.name}] start changed={len(state.changed_paths)} failed_paths={len(state.failed_paths)} " - f"checkpoint={len(checkpoint)} interest_paths={len(interest_paths)} day_indexes={len(day_index_paths)} " + f"checkpoint={len(checkpoint)} day_indexes={len(day_index_paths)} " f"deleted={len(state.deleted_paths)} persist={self.persist}", ) upserts = self._nodes(workspace, upsert_paths) @@ -72,7 +71,6 @@ class DreamFinishStep(BaseStep): def render_summary(state: DreamState) -> str: """Render a concise user-facing summary.""" - interest_paths = state.interests_paths or ([state.interests_path] if state.interests_path else []) dates = ", ".join(state.dates or [state.date]) lines = [ ("AutoDream completed with warnings" if state.warnings else "AutoDream completed"), @@ -83,12 +81,11 @@ def render_summary(state: DreamState) -> str: f"- Files: {state.files_scanned} scanned, {state.files_changed} changed, " f"{state.files_unchanged} unchanged, {state.files_deleted} deleted" ), - f"- Extracted: {len(state.units)} unit(s), {len(state.topics)} topic candidate(s)", + f"- Extracted: {len(state.units)} unit(s)", ( f"- Integrated: {len(state.integrate_results)} ok, {len(state.skipped_units)} skipped, " f"{len(state.failed_units)} failed" ), - f"- Topics: {state.topics_written} written" + (f" to {', '.join(interest_paths)}" if interest_paths else ""), f"- Catalog: checkpointed {len(state.checkpoint_paths)} changed path(s)", ] if state.nodes_created: diff --git a/reme/steps/evolve/dream/proactive.py b/reme/steps/evolve/dream/proactive.py deleted file mode 100644 index 7463970f..00000000 --- a/reme/steps/evolve/dream/proactive.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Read daily interests.yaml for proactive use.""" - -from ...base_step import BaseStep -from ....components import R -from ....schema import ProactiveResult -from .utils import load_yaml_topics, today, workspace_dir - - -@R.register("proactive_step") -class ProactiveStep(BaseStep): - """Read ``daily//interests.yaml``.""" - - def __init__(self, include_content: bool = True, **kwargs): - super().__init__(**kwargs) - self.include_content = include_content - - async def execute(self): - assert self.context is not None - day = today(self, str(self.context.get("date", "") or "")) - include_content = bool(self.context.get("include_content", self.include_content)) - daily = self.config_value("daily_dir") - rel_path, abs_path = f"{daily}/{day}/interests.yaml", workspace_dir(self) / daily / day / "interests.yaml" - result = ProactiveResult(date=day, path=rel_path) - self.logger.info(f"[{self.name}] start date={day} path={rel_path} include_content={include_content}") - - if not abs_path.is_file(): - result.skipped, result.summary = True, f"Skipped: interests file not found at {rel_path}" - self.logger.info(f"[{self.name}] skip missing path={rel_path}") - return self._finish(True, result, include_content=include_content) - try: - self.logger.info(f"[{self.name}] read start path={rel_path}") - result.content = abs_path.read_text(encoding="utf-8") if include_content else "" - result.topics = load_yaml_topics(abs_path) - self.logger.info( - f"[{self.name}] read done path={rel_path} topics={len(result.topics)} chars={len(result.content)}", - ) - except Exception as e: # noqa: BLE001 - result.error, result.summary = f"{type(e).__name__}: {e}", "" - self.logger.error(f"[{self.name}] read failed path={rel_path}: {result.error}") - return self._finish(False, result, include_content=include_content) - - result.summary = f"Read {len(result.topics)} proactive topic(s) from {rel_path}" - return self._finish(True, result, include_content=include_content) - - def _finish(self, success: bool, result: ProactiveResult, *, include_content: bool): - assert self.context is not None - self.context.response.success = success - if not success: - self.context.response.answer = f"Error: {result.error}" - elif result.skipped: - self.context.response.answer = result.summary - else: - self.context.response.answer = { - "summary": result.summary, - "topics": result.topics, - **({"content": result.content} if include_content else {}), - } - self.context.response.metadata.update(result.model_dump()) - self.logger.info(f"[{self.name}] finish success={success} answer={self.context.response.answer!r}") - return self.context.response diff --git a/reme/steps/evolve/dream/topics.py b/reme/steps/evolve/dream/topics.py deleted file mode 100644 index 9fbea087..00000000 --- a/reme/steps/evolve/dream/topics.py +++ /dev/null @@ -1,229 +0,0 @@ -"""Daily interests.yaml step.""" - -import json -from pathlib import Path - -from ...base_step import BaseStep -from ...file_io import refresh_day_index -from .._evolve import agent_reply_result_text -from ....components import R -from .utils import ( - load_yaml_topics, - llm_available, - normalize_topic, - parse_structured_reply, - previous_dates, - state_from_context, - store_state, - workspace_dir, - write_yaml, -) - - -@R.register("dream_topics_step") -class DreamTopicsStep(BaseStep): - """Write ``daily//interests.yaml`` with same-day and recent de-dup.""" - - def __init__(self, topic_count: int = 3, topic_diversity_days: int = 7, **kwargs): - super().__init__(**kwargs) - self.topic_count = topic_count - self.topic_diversity_days = topic_diversity_days - - async def execute(self): - assert self.context is not None - state = state_from_context(self) - topic_count = int(self.context.get("topic_count", self.topic_count) or self.topic_count) - raw_days = self.context.get("topic_diversity_days", self.topic_diversity_days) - diversity_days = int(raw_days or self.topic_diversity_days) - workspace = Path(state.workspace).resolve() if state.workspace else workspace_dir(self) - target_day = state.date or ((state.dates or [""])[-1]) - self.logger.info( - f"[{self.name}] start target_day={target_day!r} candidates={len(state.topics)} " - f"topic_count={topic_count} diversity_days={diversity_days}", - ) - - if not state.topics: - existing_paths = [] - if target_day and self._abs_path(workspace, state.daily_dir, target_day).is_file(): - existing_paths = [self._rel_path(state.daily_dir, target_day)] - state.interests_paths = existing_paths - state.interests_path = existing_paths[-1] if existing_paths else "" - state.topics_written = ( - len(load_yaml_topics(self._abs_path(workspace, state.daily_dir, target_day))) if target_day else 0 - ) - answer = ( - f"Kept existing interest topic(s) at {', '.join(existing_paths)}" - if existing_paths - else "Skipped interests.yaml write: no new topic candidates" - ) - self.logger.info(f"[{self.name}] skip no candidates existing_paths={len(existing_paths)}") - return self._finish(state, True, answer) - - try: - if not target_day: - state.interests_paths = [] - state.interests_path = "" - state.topics_written = 0 - self.logger.info(f"[{self.name}] skip no target date") - return self._finish(state, True, "Skipped interests.yaml write: no target date") - - rel_path = self._rel_path(state.daily_dir, target_day) - abs_path = self._abs_path(workspace, state.daily_dir, target_day) - same_day = load_yaml_topics(abs_path, strict=True) - recent = [ - topic - for previous_day in previous_dates(target_day, diversity_days) - for topic in load_yaml_topics(self._abs_path(workspace, state.daily_dir, previous_day)) - ] - self.logger.info( - f"[{self.name}] loaded context same_day={len(same_day)} recent={len(recent)} target={rel_path}", - ) - topics, _used_llm = await self._select_topics( - target_day, - state.topics, - same_day, - recent, - topic_count, - diversity_days, - state, - ) - self.logger.info(f"[{self.name}] selected topics={len(topics)} used_llm={_used_llm}") - payload = { - "date": target_day, - "topic_count": topic_count, - "diversity_days": diversity_days, - "topics": topics, - } - before_content = abs_path.read_bytes() if abs_path.is_file() else None - self.logger.info(f"[{self.name}] write yaml start path={rel_path}") - write_yaml(abs_path, payload) - self.logger.info(f"[{self.name}] write yaml done path={rel_path}") - if before_content != abs_path.read_bytes() and rel_path not in state.modified_paths: - state.modified_paths.append(rel_path) - self.logger.info(f"[{self.name}] refresh index start date={target_day} daily_dir={state.daily_dir}") - await refresh_day_index(self.file_store, target_day, state.daily_dir) - self.logger.info(f"[{self.name}] refresh index done date={target_day}") - state.interests_paths = [rel_path] - state.interests_path = rel_path - state.topics_written = len(topics) - answer = f"Wrote {len(topics)} interest topic(s) to {rel_path}" - return self._finish(state, True, answer) - except Exception as e: # noqa: BLE001 - state.topic_error = f"{type(e).__name__}: {e}" - state.errors.append(state.topic_error) - self.logger.error(f"[{self.name}] failed: {state.topic_error}") - return self._finish(state, False, f"Error: {state.topic_error}") - - async def _select_topics( - self, - day: str, - candidates: list[dict], - same_day: list[dict], - recent: list[dict], - count: int, - days: int, - state, - ): - if not candidates: - return self._dedupe([], same_day, recent, count), False - if not llm_available(self): - self.logger.info(f"[{self.name}] select topics without llm candidates={len(candidates)}") - return self._dedupe(candidates, same_day, recent, count), False - self.logger.info( - f"[{self.name}] topics agent start candidates={len(candidates)} " - f"same_day={len(same_day)} recent={len(recent)}", - ) - message = self.prompt_format( - "topics_user_message", - date=day, - topic_count=count, - diversity_days=days, - candidates_json=json.dumps(candidates, ensure_ascii=False, indent=2), - same_day_json=json.dumps(same_day, ensure_ascii=False, indent=2), - recent_topics_json=json.dumps(recent, ensure_ascii=False, indent=2), - ) - try: - result = await self.agent_wrapper.reply( - message, - system_prompt=self.prompt_format("topics_system_prompt"), - ) - self.logger.info(f"[{self.name}] topics agent done has_result={bool(result.get('result'))}") - raw_result = agent_reply_result_text(result) - meta = parse_structured_reply(raw_result) - except Exception as e: # noqa: BLE001 - warning = f"topic selection agent unavailable; used deterministic fallback ({type(e).__name__})" - state.warnings.append(warning) - self.logger.warning(f"[{self.name}] {warning}: {e}") - return self._dedupe(candidates, same_day, recent, count), False - allowed_paths = { - str(path).strip() for candidate in candidates for path in candidate.get("paths") or [] if str(path).strip() - } - selected = [self._clean_topic(t, allowed_paths) for t in meta.get("topics") or []] - if not any(selected): - if not isinstance(meta.get("topics"), list): - warning = "topic selection skipped unusable agent receipt; used deterministic fallback" - state.warnings.append(warning) - self.logger.warning(f"[{self.name}] {warning}") - self.logger.info(f"[{self.name}] topics agent produced no usable topics; fallback to candidates") - selected = candidates - return self._dedupe(selected, same_day, recent, count), True - - @staticmethod - def _rel_path(daily_dir: str, day: str) -> str: - return f"{daily_dir}/{day}/interests.yaml" - - @staticmethod - def _abs_path(workspace: Path, daily_dir: str, day: str) -> Path: - return workspace / daily_dir / day / "interests.yaml" - - @staticmethod - def _clean_topic(raw, allowed_paths: set[str] | None = None) -> dict: - if not isinstance(raw, dict): - return {} - title, reason = ( - str(raw.get("title") or "").strip(), - str(raw.get("reason") or "").strip(), - ) - if not title or not reason: - return {} - keywords, paths = raw.get("keywords") or [], raw.get("paths") or [] - cleaned_keywords = [str(k).strip() for k in keywords if str(k).strip()] if isinstance(keywords, list) else [] - cleaned_paths = ( - [ - str(p).strip() - for p in paths - if str(p).strip() and (allowed_paths is None or str(p).strip() in allowed_paths) - ] - if isinstance(paths, list) - else [] - ) - return { - "title": title, - "reason": reason, - "evidence": str(raw.get("evidence") or "").strip(), - "keywords": cleaned_keywords, - "paths": cleaned_paths, - } - - @staticmethod - def _dedupe(topics: list[dict], same_day: list[dict], recent: list[dict], count: int) -> list[dict]: - recent_norm = {normalize_topic(t.get("title", "")) for t in recent} - seen = {normalize_topic(t.get("title", "")) for t in same_day} - out = list(same_day) - for topic in [t for t in topics if t]: - title_norm = normalize_topic(topic.get("title", "")) - if title_norm and title_norm not in seen and title_norm not in recent_norm: - seen.add(title_norm) - out.append(topic) - if len(out) >= count: - break - return out[:count] - - def _finish(self, state, success: bool, answer: str): - assert self.context is not None - state.summary = answer - store_state(self, state) - self.context.response.success = success - self.context.response.answer = answer - self.logger.info(f"[{self.name}] finish success={success} answer={answer!r}") - return self.context.response diff --git a/reme/steps/evolve/dream/topics.yaml b/reme/steps/evolve/dream/topics.yaml deleted file mode 100644 index 8e3115db..00000000 --- a/reme/steps/evolve/dream/topics.yaml +++ /dev/null @@ -1,107 +0,0 @@ -topics_system_prompt: | - You select final daily user-interest topics from dream candidates for - daily//interests.yaml. - - ## Goals - - - Preserve existing same-day topics unless they are clear duplicates. - - Avoid duplicates within today. - - Avoid topics already covered in recent interests.yaml files. - - Prefer concrete, recurring, or actionable interests over file summaries. - - Return no more than topic_count topics. - - ## Topic Quality - - A good topic is something the user may want surfaced again: a live area of - attention, a recurring concern, a research direction, a project thread, or a - practical follow-up. Avoid generic labels, daily log summaries, and topics - whose only value is restating a file name. - - ## Output Rules - - - Keep titles concise and specific. - - Keep reason grounded in candidate evidence. - - Keep keywords short and retrieval-friendly. - - Keep paths limited to the paths supplied by the candidates. - - Quote scalar strings that contain punctuation such as `:` or use block - scalars (`>`). Write paths as block lists, not flow lists, so - `daily//...` stays parseable. - - Return only one YAML or JSON object: - topics: - - title: - reason: - evidence: - keywords: [, ...] - paths: [, ...] - -topics_system_prompt_zh: | - 你负责从 dream 候选中选择最终 daily user-interest topics，用于写入 daily//interests.yaml。 - - ## 目标 - - - 保留同一天已有的 topics，除非它们明显重复。 - - 避免当天内部重复。 - - 避免重复最近 interests.yaml 已经覆盖过的 topics。 - - 优先选择具体、反复出现、可行动的兴趣，而不是文件摘要。 - - 返回数量不能超过 topic_count。 - - ## Topic 质量 - - 好的 topic 应该是用户未来可能希望再次看到的内容：持续关注的方向、反复出现的问题、研究方向、 - 项目线索或实际 follow-up。避免泛泛标签、daily log 摘要，以及只是在复述文件名的 topic。 - - ## 输出规则 - - - title 要简短、具体。 - - reason 要基于候选证据。 - - keywords 要短，方便检索。 - - paths 只能使用候选中提供的 paths。 - - 如果字符串包含 `:` 等标点，要加引号或使用 block scalar (`>`)。 - paths 使用 block list，不要用 flow list，避免 `daily//...` 解析失败。 - - 只返回一个 YAML 或 JSON object： - topics: - - title: <具体 topic> - reason: <为什么重要> - evidence: <简短证据指针> - keywords: [<关键词>, ...] - paths: [, ...] - -topics_user_message: | - date: {date} - topic_count: {topic_count} - diversity_days: {diversity_days} - - # Candidate topics - - {candidates_json} - - # Existing same-day interests.yaml topics - - {same_day_json} - - # Recent interests.yaml topics to avoid repeating - - {recent_topics_json} - - Select the final topics for daily interests.yaml. - -topics_user_message_zh: | - 日期：{date} - topic_count: {topic_count} - diversity_days: {diversity_days} - - # 候选 topics - - {candidates_json} - - # 同一天已有的 interests.yaml topics - - {same_day_json} - - # 需要避免重复的最近 interests.yaml topics - - {recent_topics_json} - - 选择最终要写入 daily interests.yaml 的 topics。 diff --git a/reme/steps/evolve/dream/utils.py b/reme/steps/evolve/dream/utils.py index 18dc70df..7701d005 100644 --- a/reme/steps/evolve/dream/utils.py +++ b/reme/steps/evolve/dream/utils.py @@ -3,7 +3,6 @@ import datetime as dt import re from pathlib import Path -from uuid import uuid4 import yaml @@ -67,8 +66,8 @@ def llm_available(step: BaseStep) -> bool: return False -def scan_day_files(workspace: Path, day: str, daily: str, interests_name: str = "interests.yaml") -> list[str]: - """Scan day files.""" +def scan_day_files(workspace: Path, day: str, daily: str) -> list[str]: + """Scan Markdown day-index and note files.""" out: list[str] = [] day_index = workspace / daily / f"{day}.md" if day_index.is_file(): @@ -76,24 +75,42 @@ def scan_day_files(workspace: Path, day: str, daily: str, interests_name: str = daily_root = workspace / daily / day if daily_root.is_dir(): out.extend(p.relative_to(workspace).as_posix() for p in sorted(daily_root.rglob("*.md")) if p.is_file()) - return [p for p in out if p != f"{daily}/{day}/{interests_name}"] + return out -def pack_paths(workspace: Path, paths: list[str], *, limit_per_file: int = 60000) -> str: - """Pack paths into a single string.""" +def pack_paths( + workspace: Path, + paths: list[str], + *, + limit_per_file: int = 60000, + max_total_chars: int | None = None, +) -> str: + """Pack paths into a single string. + + With ``max_total_chars`` set, blocks are packed in the given order until + the accumulated size would exceed the budget; the first file is always + kept and a trailer records how many files were omitted. + """ blocks: list[str] = [] - for rel in paths: + total = 0 + for index, rel in enumerate(paths): target = workspace / rel if not target.is_file(): - blocks.append(f"### {rel}\n(file not found)\n") - continue - try: - text = target.read_text(encoding="utf-8") - except Exception as e: # noqa: BLE001 - blocks.append(f"### {rel}\n(error reading: {type(e).__name__}: {e})\n") - continue - suffix = "\n\n[truncated]\n" if len(text) > limit_per_file else "" - blocks.append(f"### {rel}\n{text[:limit_per_file]}{suffix}\n") + block = f"### {rel}\n(file not found)\n" + else: + try: + text = target.read_text(encoding="utf-8") + except Exception as e: # noqa: BLE001 + block = f"### {rel}\n(error reading: {type(e).__name__}: {e})\n" + else: + suffix = "\n\n[truncated]\n" if len(text) > limit_per_file else "" + block = f"### {rel}\n{text[:limit_per_file]}{suffix}\n" + if max_total_chars is not None and index > 0 and total + len(block) > max_total_chars: + omitted = len(paths) - index + blocks.append(f"(omitted {omitted} file(s) to stay within the {max_total_chars}-char total budget)") + break + blocks.append(block) + total += len(block) return "\n".join(blocks) @@ -109,11 +126,6 @@ def clean_paths(raw_paths, allowed: set[str]) -> list[str]: return out -def normalize_topic(text: str) -> str: - """Normalize topic.""" - return re.sub(r"[^a-z0-9\u4e00-\u9fff]+", " ", text.lower()).strip() - - def previous_dates(day: str, n_days: int) -> list[str]: """Get previous dates.""" try: @@ -123,90 +135,6 @@ def previous_dates(day: str, n_days: int) -> list[str]: return [(base - dt.timedelta(days=i)).isoformat() for i in range(1, max(n_days, 0) + 1)] -def load_yaml_topics(path: Path, *, strict: bool = False) -> list[dict]: - """Load YAML topics.""" - if not path.is_file(): - return [] - try: - data = yaml.safe_load(path.read_text(encoding="utf-8")) - except Exception as exc: - if strict: - raise ValueError(f"Invalid interests YAML at {path}: {exc}") from exc - return [] - if data is None: - if strict: - raise ValueError(f"Invalid interests YAML at {path}: expected an object") - return [] - topics = data.get("topics") if isinstance(data, dict) else None - if not isinstance(topics, list): - if strict: - raise ValueError(f"Invalid interests YAML at {path}: topics must be a list") - return [] - cleaned_topics = [] - for index, topic in enumerate(topics): - if strict: - _validate_topic(topic, path, index) - if isinstance(topic, dict) and (cleaned := clean_topic(topic)): - cleaned_topics.append(cleaned) - return cleaned_topics - - -def _validate_topic(topic: object, path: Path, index: int) -> None: - """Reject topic data that would otherwise be silently discarded or coerced.""" - prefix = f"Invalid interests YAML at {path}: topics[{index}]" - if not isinstance(topic, dict): - raise ValueError(f"{prefix} must be an object") - - allowed = {"title", "reason", "evidence", "keywords", "paths"} - if unknown := sorted(set(topic) - allowed): - raise ValueError(f"{prefix} has unknown field(s): {', '.join(str(key) for key in unknown)}") - - for field in ("title", "reason"): - value = topic.get(field) - if not isinstance(value, str) or not value.strip(): - raise ValueError(f"{prefix}.{field} must be a non-empty string") - if "evidence" in topic and not isinstance(topic["evidence"], str): - raise ValueError(f"{prefix}.evidence must be a string") - for field in ("keywords", "paths"): - if field not in topic: - continue - values = topic[field] - if not isinstance(values, list) or any(not isinstance(value, str) or not value.strip() for value in values): - raise ValueError(f"{prefix}.{field} must be a list of non-empty strings") - - -def clean_topic(raw: dict) -> dict: - """Clean topic.""" - title, reason = ( - str(raw.get("title") or "").strip(), - str(raw.get("reason") or "").strip(), - ) - if not title or not reason: - return {} - keywords = raw.get("keywords") or [] - paths = raw.get("paths") or [] - return { - "title": title, - "reason": reason, - "evidence": str(raw.get("evidence") or "").strip(), - "keywords": ([str(k).strip() for k in keywords if str(k).strip()] if isinstance(keywords, list) else []), - "paths": ([str(p).strip() for p in paths if str(p).strip()] if isinstance(paths, list) else []), - } - - -def write_yaml(path: Path, payload: dict) -> None: - """Atomically write YAML without exposing a partially written user file.""" - path.parent.mkdir(parents=True, exist_ok=True) - rendered = yaml.safe_dump(payload, allow_unicode=True, sort_keys=False) - content = rendered if rendered.endswith("\n") else f"{rendered}\n" - temporary = path.with_name(f".{path.name}.{uuid4().hex}.tmp") - try: - temporary.write_text(content, encoding="utf-8") - temporary.replace(path) - finally: - temporary.unlink(missing_ok=True) - - def parse_structured_reply(text: str) -> dict: """Parse a JSON/YAML object from an agent reply, including fenced blocks.""" candidates = [text.strip()] diff --git a/reme/steps/evolve/proactive/__init__.py b/reme/steps/evolve/proactive/__init__.py new file mode 100644 index 00000000..508583ae --- /dev/null +++ b/reme/steps/evolve/proactive/__init__.py @@ -0,0 +1,17 @@ +"""Proactive refresh steps: independent of the nightly dream chain.""" + +from .agenda import ProactiveAgendaStep +from .extract import ProactiveExtractStep +from .finish import ProactiveFinishStep +from .plan import ProactivePlanStep +from .proactive import ProactiveStep +from .topics import ProactiveTopicsStep + +__all__ = [ + "ProactiveAgendaStep", + "ProactiveExtractStep", + "ProactiveFinishStep", + "ProactivePlanStep", + "ProactiveStep", + "ProactiveTopicsStep", +] diff --git a/reme/steps/evolve/proactive/agenda.py b/reme/steps/evolve/proactive/agenda.py new file mode 100644 index 00000000..bd15becb --- /dev/null +++ b/reme/steps/evolve/proactive/agenda.py @@ -0,0 +1,277 @@ +"""Proactive agenda step: list-wise generative agenda + auditable silence (F2.6). + +Replaces "sort and truncate" selection with generation: the LLM sees every +scenario card plus today's conversation context and an optional user profile, +then decides BY ITSELF how many items deserve today's agenda (bounded only by +the ``max_agenda_items`` safety cap) and must explicitly account for every +candidate it does not schedule (suppressed with a reason - silence is +auditable). This lets it merge candidates that are different entrances to the +same matter, keep narrative coherence, and avoid clashing with the user's +current focus. + +Deterministic guarantees: exactly one candidate -> auto-agenda with no LLM +call; LLM absent/timed out/unusable -> freshness/confidence order fallback; +after validation the agenda is never empty while candidates exist, so the push +semantics computed by the topics step never flip. The step also owns the final +interests.yaml render, extending the v2 shape with ``agenda``/``suppressed``. +""" + +import asyncio +import json +import time + +from ...base_step import BaseStep +from .._evolve import agent_reply_result_text, passthrough_response +from ....components import R +from ....schema import ProactiveState +from ..dream.utils import daily_dir, workspace_dir +from .utils import ( + current_now, + interests_path_for, + load_personal_profile_block, + norm_path, + parse_agenda_reply, + render_interests, + resolve_agent_wrapper, + write_interests_if_changed, +) + +FALLBACK_ORDER_REASON = "deterministic fallback: freshness/confidence order" + + +@R.register("proactive_agenda_step") +class ProactiveAgendaStep(BaseStep): + """Generate today's proactive agenda and render the enriched interests.yaml. + + Skips silently (leaving the topics step's push=false render in place) when + there are no push candidates. Otherwise writes the final v2 file with the + ``agenda`` and ``suppressed`` keys. + """ + + def __init__( + self, + max_agenda_items: int = 6, + llm_timeout_seconds: float = 120, + profile_rel_path: str = "profile.md", + profile_max_chars: int = 2500, + skip_key: str = "proactive_skip", + **kwargs, + ): + super().__init__(**kwargs) + self.max_agenda_items = max(int(max_agenda_items), 1) + self.llm_timeout_seconds = float(llm_timeout_seconds) + self.profile_rel_path = str(profile_rel_path or "") + self.profile_max_chars = max(int(profile_max_chars), 0) + self.skip_key = skip_key + + async def execute(self): + assert self.context is not None + if self.context.get(self.skip_key): + return passthrough_response(self, self.skip_key) + started = time.monotonic() + raw_state = self.context.get("proactive") + if not raw_state: + self.context.response.success = True + self.context.response.answer = "Skipped agenda: no proactive extract state in context" + return self.context.response + state = ProactiveState.model_validate(raw_state) + if state.early_exit: + self.context.response.success = True + self.context.response.answer = f"Skipped agenda: {state.early_exit}" + return self.context.response + + candidates = [c for c in state.push_candidates if isinstance(c, dict) and c.get("id")] + if not candidates: + self.context.response.success = True + self.context.response.answer = "Agenda: no push candidates; interests.yaml left to topics render" + self.logger.info(f"[{self.name}] no push candidates; nothing to agenda") + return self.context.response + + ws = workspace_dir(self) + cards_by_id = {str(c.get("topic_id") or ""): c for c in state.scenario_cards if isinstance(c, dict)} + carded = [c for c in candidates if str(c.get("id")) in cards_by_id] + over_budget = [c for c in candidates if str(c.get("id")) not in cards_by_id] + self.logger.info( + f"[{self.name}] start candidates={len(candidates)} carded={len(carded)} " + f"over_budget={len(over_budget)} max_agenda_items={self.max_agenda_items}", + ) + + if not carded: + agenda_ids, order_reasons, suppressed_reasons = [], {}, {} + elif len(carded) == 1: + agenda_ids = [str(carded[0].get("id"))] + order_reasons = {agenda_ids[0]: "single push candidate; auto-agenda without LLM"} + suppressed_reasons = {} + else: + agenda_ids, order_reasons, suppressed_reasons = await self._llm_agenda(state, ws, carded, cards_by_id) + + suppressed: list[dict] = [] + for candidate in over_budget: + suppressed.append( + { + "topic_id": str(candidate.get("id")), + "title": str(candidate.get("title") or ""), + "reason": "over plan budget: not expanded into a scenario card", + }, + ) + for candidate in carded: + cid = str(candidate.get("id")) + if cid in agenda_ids: + continue + suppressed.append( + { + "topic_id": cid, + "title": str(candidate.get("title") or ""), + "reason": suppressed_reasons.get(cid) or "not selected for today's agenda", + }, + ) + + agenda: list[dict] = [] + for cid in agenda_ids: + card = cards_by_id.get(cid) + candidate = next((c for c in carded if str(c.get("id")) == cid), None) + if card is None or candidate is None: + continue + agenda.append( + { + "topic_id": cid, + "title": str(candidate.get("title") or ""), + "scenario_type": str(card.get("scenario_type") or ""), + "opener": str(card.get("opener") or ""), + "next_action": str(card.get("next_action") or ""), + "preconditions": list(card.get("preconditions") or []), + "delivery": str(card.get("delivery") or "in_conversation"), + "linked_memory": list(card.get("linked_memory") or []), + "order_reason": order_reasons.get(cid) or "", + }, + ) + + state.agenda = agenda + state.suppressed = suppressed + state.push = bool(agenda) + + day = state.date + daily = state.daily_dir or daily_dir(self) + rendered = render_interests( + day, + state.topics_out, + state.push, + current_now(self), + agenda=agenda, + suppressed=suppressed, + ) + interests_path = interests_path_for(ws, daily, day) + written = write_interests_if_changed(ws, interests_path, rendered) + rel_path = norm_path(interests_path.relative_to(ws).as_posix()) + state.interests_path = rel_path + state.interests_written = bool(state.interests_written or written) + state.duration_ms = state.duration_ms + int((time.monotonic() - started) * 1000) + self._store(state) + answer = ( + f"Agenda: {len(agenda)} item(s), suppressed={len(suppressed)}, push={state.push}, " + f"written={written} to {rel_path}" + ) + self.context.response.success = True + self.context.response.answer = answer + self.logger.info(f"[{self.name}] finish {answer}") + return self.context.response + + async def _llm_agenda( + self, + state: ProactiveState, + ws, + carded: list[dict], + cards_by_id: dict[str, dict], + ) -> tuple[list[str], dict[str, str], dict[str, str]]: + """One list-wise LLM call; returns (agenda ids, order reasons, suppressed reasons).""" + carded_ids = [str(c.get("id")) for c in carded] + wrapper = resolve_agent_wrapper(self) + if wrapper is None: + self.logger.warning(f"[{self.name}] no agent_wrapper available; deterministic fallback agenda") + return self._fallback_agenda(carded_ids) + + cards_json = json.dumps( + [ + { + "topic_id": str(c.get("id")), + "title": c.get("title"), + "kind": c.get("kind"), + "scenario_type": cards_by_id[str(c.get("id"))].get("scenario_type"), + "opener": cards_by_id[str(c.get("id"))].get("opener"), + "next_action": cards_by_id[str(c.get("id"))].get("next_action"), + "delivery": cards_by_id[str(c.get("id"))].get("delivery"), + } + for c in carded + ], + ensure_ascii=False, + ) + user_message = self.prompt_format( + "agenda_user_message", + max_agenda_items=self.max_agenda_items, + cards_json=cards_json, + changed_files_json=json.dumps(list(dict.fromkeys(state.changed_paths)), ensure_ascii=False), + profile_block=self._profile_block(ws), + ) + system_prompt = self.prompt_format("agenda_system_prompt", max_agenda_items=self.max_agenda_items) + state.plan_llm_calls += 1 + try: + result = await asyncio.wait_for( + wrapper.reply(user_message, system_prompt=system_prompt), + timeout=self.llm_timeout_seconds, + ) + except asyncio.TimeoutError: + self.logger.warning(f"[{self.name}] LLM reply timed out after {self.llm_timeout_seconds}s") + return self._fallback_agenda(carded_ids) + except Exception as e: # noqa: BLE001 - network/provider errors degrade to fallback agenda + self.logger.warning( + f"[{self.name}] LLM reply failed ({type(e).__name__}: {e}); using fallback agenda", + ) + return self._fallback_agenda(carded_ids) + raw = agent_reply_result_text(result) + agenda_raw, suppressed_raw = parse_agenda_reply(raw) + + allowed = set(carded_ids) + agenda_ids: list[str] = [] + for item in agenda_raw: + cid = str(item.get("topic_id") or "").strip() + if cid in allowed and cid not in agenda_ids: + agenda_ids.append(cid) + if len(agenda_ids) >= self.max_agenda_items: + break + order_reasons = { + str(item.get("topic_id") or "").strip(): str(item.get("order_reason") or "").strip() + for item in agenda_raw + if str(item.get("topic_id") or "").strip() in allowed + } + suppressed_reasons = { + str(item.get("topic_id") or "").strip(): str(item.get("reason") or "").strip() + for item in suppressed_raw + if str(item.get("topic_id") or "").strip() in allowed + } + if not agenda_ids: + self.logger.warning(f"[{self.name}] agenda reply unusable; raw={raw[:200]!r}") + return self._fallback_agenda(carded_ids) + return agenda_ids, order_reasons, suppressed_reasons + + def _fallback_agenda(self, carded_ids: list[str]) -> tuple[list[str], dict[str, str], dict[str, str]]: + """Freshness/confidence head when generation is unavailable (cards keep sort order).""" + agenda_ids = carded_ids[: self.max_agenda_items] + order_reasons = {cid: FALLBACK_ORDER_REASON for cid in agenda_ids} + suppressed_reasons = {cid: "capacity (deterministic fallback)" for cid in carded_ids if cid not in agenda_ids} + return agenda_ids, order_reasons, suppressed_reasons + + def _profile_block(self, ws) -> str: + """Digest-personal profile first, legacy single profile file as fallback.""" + digest_dir = str(self.config_value("digest_dir")) + return load_personal_profile_block( + ws, + digest_dir, + self.profile_max_chars, + self.profile_rel_path, + ) + + def _store(self, state: ProactiveState) -> None: + assert self.context is not None + data = state.model_dump() + self.context["proactive"] = data + self.context.response.metadata["proactive"] = data diff --git a/reme/steps/evolve/proactive/agenda.yaml b/reme/steps/evolve/proactive/agenda.yaml new file mode 100644 index 00000000..e67283fd --- /dev/null +++ b/reme/steps/evolve/proactive/agenda.yaml @@ -0,0 +1,94 @@ +agenda_system_prompt: | + You are the editor of today's proactive agenda for a personal memory + assistant. You receive scenario cards (already expanded from the push + candidates), the files that changed today (a proxy for what the user is + focused on right now) and an optional user profile. + + Decide which cards enter today's agenda, in which order, and which are + held back. The number of agenda items is YOUR call: judge it from content + richness and redundancy - anything from 1 up to at most {max_agenda_items}. + Fewer, stronger items beat many weak ones. + + ## Editorial rules + + - Merge, don't stack: when two cards are really the same matter reached + from different entrances, keep the single better entrance on the agenda + and suppress the other with reason `merged into `. + - Narrative coherence: today's agenda should read as one coherent story, + not a grab bag; order items so one naturally leads to the next. + - Focus conflict: if a card clashes with what the user is clearly focused + on today (changed files / profile), suppress it and say so. + - Auditable silence: EVERY candidate topic_id must appear exactly once, + either in `agenda` or in `suppressed` (with a short reason). Never drop + a candidate without a word. + + ## Output format + + Return only one YAML fenced block: + ```yaml + agenda: + - topic_id: + order_reason: + suppressed: + - topic_id: + reason: " for merges> + ``` + +agenda_system_prompt_zh: | + 你是个人记忆助手当日主动议程的编辑。输入是场景卡（由推送候选扩展而来）、 + 今天发生变化的文件（代表用户当前关注点）以及可选的用户画像。 + + 决定哪些卡片进入今日议程、按什么顺序，哪些暂缓。议程条数由你自行裁决： + 依据内容丰富度和重复程度决定，最少 1 条，至多不超过 {max_agenda_items} 条。 + 少而强胜过多而弱。 + + ## 编辑规则 + + - 合并不堆砌：若两张卡其实是同一件事的不同入口，只保留更好的那个入口 + 进入议程，另一个放入 suppressed，reason 写 `merged into <保留的 topic_id>`。 + - 叙事连贯：今日议程应读起来是一条连贯的线索，而不是一堆碎片；排序让 + 前一项能自然引出后一项。 + - 关注点冲突：若某张卡与用户今天明显关注的事情（变化文件/画像）冲突， + 压制它并在 reason 中说明。 + - 沉默可审计：每个候选 topic_id 必须恰好出现一次，要么在 agenda，要么在 + suppressed（附简短理由）。禁止不声不响地丢弃任何候选。 + + ## 输出格式 + + 只返回一个 YAML fenced block： + ```yaml + agenda: + - topic_id: + order_reason: <为什么放这里、为什么是现在> + suppressed: + - topic_id: + reason: <简短理由；合并用 "merged into "> + ``` + +agenda_user_message: | + max_agenda_items: {max_agenda_items} + + scenario_cards (candidate set): + {cards_json} + + files changed today (current focus hint): + {changed_files_json} + + user_profile: + {profile_block} + + Emit the agenda/suppressed block per the contract. + +agenda_user_message_zh: | + 议程条数上限：{max_agenda_items} + + 场景卡（候选集）： + {cards_json} + + 今日变化文件（当前关注点提示）： + {changed_files_json} + + 用户画像： + {profile_block} + + 按契约输出 agenda/suppressed block。 diff --git a/reme/steps/evolve/proactive/extract.py b/reme/steps/evolve/proactive/extract.py new file mode 100644 index 00000000..3c8466b7 --- /dev/null +++ b/reme/steps/evolve/proactive/extract.py @@ -0,0 +1,266 @@ +"""Proactive refresh extract step: material scan + follow_ups/extends/updates (F2.0-F2.3).""" + +import asyncio +import json +import time + +from ...base_step import BaseStep +from .._evolve import agent_reply_result_text, passthrough_response +from ....components import R +from ....schema import ProactiveState +from ..dream.utils import daily_dir, pack_paths, today, workspace_dir +from .utils import ( + clean_candidate, + load_carry_forward, + load_personal_profile_block, + load_state, + parse_extract_reply, + resolve_agent_wrapper, + scan_material_daily, +) + + +@R.register("proactive_extract_step") +class ProactiveExtractStep(BaseStep): + """Scan the proactive material set and extract follow-ups, extends and updates. + + Owns the ``proactive`` catalog watermark (never touches the dream catalog) + and the daily LLM budget. Short-circuits via ``context[skip_key]`` on + busy/budget/timeout per F4.3; early-exits with zero LLM calls when no new + evidence exists (F2.0). + """ + + def __init__( + self, + scan_days: int = 2, + carry_forward_days: int = 14, + max_carry_forward_topics: int = 20, + llm_timeout_seconds: float = 300, + max_chars_per_file: int = 60000, + max_total_chars: int = 300000, + profile_max_chars: int = 1500, + extends_enabled: bool = True, + skip_key: str = "proactive_skip", + **kwargs, + ): + super().__init__(**kwargs) + self.scan_days = max(int(scan_days), 1) + self.carry_forward_days = max(int(carry_forward_days), 1) + self.max_carry_forward_topics = max(int(max_carry_forward_topics), 0) + self.llm_timeout_seconds = float(llm_timeout_seconds) + self.max_chars_per_file = max(int(max_chars_per_file), 1000) + self.max_total_chars = max(int(max_total_chars), 0) + self.profile_max_chars = max(int(profile_max_chars), 0) + self.extends_enabled = bool(extends_enabled) + self.skip_key = skip_key + + async def execute(self): + assert self.context is not None + if self.context.get(self.skip_key): + return passthrough_response(self, self.skip_key) + started = time.monotonic() + day = today(self, str(self.context.get("date", "") or "")) + ws = workspace_dir(self) + daily = daily_dir(self) + if self.file_catalog is None: + raise RuntimeError("proactive_extract_step requires file_catalog") + state = ProactiveState( + date=day, + daily_dir=daily, + workspace=str(ws), + scan_days=self.scan_days, + carry_forward_days=self.carry_forward_days, + ) + self.logger.info(f"[{self.name}] start date={day} scan_days={self.scan_days} extends={self.extends_enabled}") + + # 1) Material set M (F2.0) - all cheap, before any LLM work. + m_daily = scan_material_daily(ws, day, daily, self.scan_days) + + # 2) Truth source + carry-forward (F1.3/F1.4). + state_file, needs_bootstrap = load_state(ws, daily) + carry_all, carry_prompt = await load_carry_forward( + ws, + state_file, + day, + self.carry_forward_days, + self.max_carry_forward_topics, + daily, + needs_bootstrap, + ) + state.carry_forward_count = len(carry_all) + state.carry_forward_prompt = carry_prompt + + # 3) Change detection against the proactive catalog watermark. + # Only daily notes are material: fresh resource uploads already flow + # into daily notes via auto_resource, so no separate resource scan. + existing = {} + for rel in m_daily: + try: + existing[rel] = (ws / rel).stat().st_mtime + except OSError as e: + self.logger.error(f"[{self.name}] stat failed on {rel}: {e}") + nodes = await self.file_catalog.get_nodes() + indexed = {n.path: n.st_mtime for n in nodes} + state.changed_paths = [rel for rel, mt in existing.items() if indexed.get(rel) != mt] + # mtime snapshot of what this round actually read; finish only + # checkpoints paths whose mtime still matches, so files modified while + # the LLM calls run stay "changed" for the next round (audit item 3). + state.changed_mtimes = {rel: existing[rel] for rel in state.changed_paths} + self.logger.info( + f"[{self.name}] material daily={len(m_daily)} " + f"changed={len(state.changed_paths)} carry_forward={len(carry_all)}", + ) + + # 4) Zero-consumption early exit (A4 row 1). + if not state.changed_paths: + state.early_exit = "no_new_evidence" + return self._finish(state, started, "Skipped: no new evidence; 0 LLM calls") + + # 5) LLM channel (F4.4): structurally one reply per round plus at most + # one parse-failure retry, so no persistent budget is needed (v5 R5). + wrapper = resolve_agent_wrapper(self) + if wrapper is None: + state.early_exit = "no_agent_wrapper" + self.logger.warning(f"[{self.name}] no agent_wrapper available; skipping round") + return self._finish(state, started, "Skipped: no agent_wrapper configured") + + # 6) One LLM call with sectioned output (A3); retry once on parse failure. + # meta is None when this round must NOT checkpoint (timeout, or two + # unusable replies): the material stays "changed" for the next round + # instead of being silently consumed on output that can never yield + # topics (audit #1). + meta = await self._extract_with_retry(wrapper, state, ws, day, daily) + if meta is None: + self._store(state) + return passthrough_response(self, self.skip_key) + self._clean_output(state, meta, set(m_daily), day) + answer = ( + f"Extracted {len(state.follow_ups)} follow_up(s), {len(state.extends)} extend(s), " + f"{len(state.updates)} update(s) from {len(state.changed_paths)} changed file(s)" + ) + return self._finish(state, started, answer) + + def _build_messages(self, state: ProactiveState, ws, day: str, daily: str, material_blob: str) -> tuple[str, str]: + carry_forward_json = json.dumps( + [ + { + "id": t.id, + "title": t.title, + "kind": t.kind, + "confidence": t.confidence, + "reason": t.reason, + "last_evidence_at": t.last_evidence_at, + "evidence": t.evidence, + } + for t in state.carry_forward_prompt + ], + ensure_ascii=False, + ) + changed = list(dict.fromkeys(state.changed_paths)) + user_message = self.prompt_format( + "extract_user_message", + date=day, + changed_paths_json=json.dumps(changed, ensure_ascii=False), + carry_forward_json=carry_forward_json, + material_blob=material_blob, + profile_block=self._profile_block(ws), + extends=self.extends_enabled, + ) + system_prompt = self.prompt_format( + "extract_system_prompt", + workspace_dir=str(ws), + daily_dir=daily, + extends=self.extends_enabled, + ) + return user_message, system_prompt + + def _profile_block(self, ws) -> str: + """Digest-personal profile sketch; background knowledge, never material.""" + digest_dir = str(self.config_value("digest_dir")) + return load_personal_profile_block(ws, digest_dir, self.profile_max_chars) + + async def _extract_with_retry(self, wrapper, state, ws, day: str, daily: str) -> dict | None: + """Returns parsed meta; None means skip without checkpointing.""" + # Newest daily material first: when the total budget bites, the + # freshest evidence survives and the oldest files are omitted. + ordered = list(dict.fromkeys(state.changed_paths))[::-1] + material_blob = pack_paths( + ws, + ordered, + limit_per_file=self.max_chars_per_file, + max_total_chars=self.max_total_chars or None, + ) + user_message, system_prompt = self._build_messages(state, ws, day, daily, material_blob) + raw = "" + for attempt in (1, 2): + raw = await self._reply(wrapper, state, user_message, system_prompt) + if raw is None: + self.context[self.skip_key] = {"reason": "llm_timeout"} + return None + meta = parse_extract_reply(raw) + if meta: + return meta + self.logger.warning(f"[{self.name}] parse failed on attempt {attempt}; raw={raw[:200]!r}") + self.logger.error( + f"[{self.name}] reply unusable after 2 attempts; keeping material un-checkpointed; " f"raw={raw[:200]!r}", + ) + self.context[self.skip_key] = {"reason": "extract_parse_failed"} + return None + + async def _reply(self, wrapper, state, user_message, system_prompt): + """One timeout-wrapped reply; returns raw text or None on timeout.""" + state.llm_calls += 1 + try: + result = await asyncio.wait_for( + wrapper.reply(user_message, system_prompt=system_prompt), + timeout=self.llm_timeout_seconds, + ) + except asyncio.TimeoutError: + self.logger.warning(f"[{self.name}] LLM reply timed out after {self.llm_timeout_seconds}s") + return None + return agent_reply_result_text(result) + + def _clean_output(self, state: ProactiveState, meta: dict, allowed: set[str], day: str) -> None: + for raw in meta.get("follow_ups") or []: + if candidate := clean_candidate(raw, allowed, "follow_up", day): + state.follow_ups.append(candidate) + else: + state.dropped_missing += 1 + if self.extends_enabled: + for raw in meta.get("extends") or []: + if candidate := clean_candidate(raw, allowed, "interest_extend", day): + state.extends.append(candidate) + else: + state.dropped_missing += 1 + for raw in meta.get("updates") or []: + if not isinstance(raw, dict): + continue + topic_id = str(raw.get("id") or "").strip() + if not topic_id: + continue + action = str(raw.get("action") or "keep").strip() + if action not in ("keep", "update", "resolve"): + action = "keep" + state.updates.append( + { + "id": topic_id, + "action": action, + "evidence": str(raw.get("evidence") or "").strip()[:120], + "reason": str(raw.get("reason") or "").strip(), + "confidence": raw.get("confidence"), + }, + ) + + def _store(self, state: ProactiveState) -> None: + assert self.context is not None + data = state.model_dump() + self.context["proactive"] = data + self.context.response.metadata["proactive"] = data + + def _finish(self, state: ProactiveState, started: float, answer: str): + state.duration_ms = int((time.monotonic() - started) * 1000) + self._store(state) + self.context.response.success = True + self.context.response.answer = answer + self.logger.info(f"[{self.name}] finish answer={answer!r}") + return self.context.response diff --git a/reme/steps/evolve/proactive/extract.yaml b/reme/steps/evolve/proactive/extract.yaml new file mode 100644 index 00000000..0319ffdb --- /dev/null +++ b/reme/steps/evolve/proactive/extract.yaml @@ -0,0 +1,211 @@ +extract_system_prompt: | + You are the proactive discovery agent for a personal memory workspace. + You read recently changed conversation notes and maintain a small set of + user-interest topics. You never browse the web, + never write files, and never collect material yourself. + + workspace_dir: {workspace_dir} + + ## Branch A - follow_ups (open loops) + + Find unresolved matters the user may still want to close: + - questions that were asked but never answered; + - tasks that were started, interrupted, or postponed; + - commitments or plans mentioned without any follow-through yet. + Emit each as a `follow_ups` entry. + + [extends]## Branch B - extends (interest boundary) + [extends] + [extends]Infer topics the user has NOT focused on yet, but that are plausibly + [extends]relevant to their recent work, using the conversation notes. These + [extends]guide future knowledge-source expansion; you only describe them, you + [extends]never collect anything. + [extends]Emit each as an `extends` entry. + + ## updates (carried-forward topics) + + For every topic in carry_forward_topics, decide exactly one action: + - `keep`: still open but no new evidence today; + - `update`: new evidence appeared today; refresh evidence and confidence + (emit the re-scored confidence value); + - `resolve`: the conversation shows the matter is settled or done. + You must echo the given `id` unchanged. Never invent ids, and never emit an + update for a topic that is not in carry_forward_topics. + If two carry_forward topics turn out to be the same matter in different + words, keep the one with stronger evidence and `resolve` the other one + (note ``merged into `` in reason). + Only `action=update` when the fresh evidence lives in a file listed in this + round's changed_paths; otherwise emit `keep` (updates citing anything else + are rejected downstream). + + ## Confidence rubric (fixed; do not freestyle scores) + + - 0.9: explicit unfinished action with a time or commitment; + - 0.7: the user stated intent explicitly but no concrete action yet; + - 0.5: inferential link from several weak signals; + - 0.3: weak association only; + - below 0.3: do not output the topic at all. + + ## Hard rules + + - Empty lists are a normal, expected output. Never invent topics to fill + space, and never promote a one-off weak mention into a topic. + - Every topic must be traceable to this round's material: `paths` may only + contain values from changed_paths (paths look like `{daily_dir}//...`); + entries violating this are discarded. + - Do not restate topics already listed in carry_forward_topics; use + `updates` to change their status instead. + - Same-matter rule: if a finding is really a carry_forward topic in + different words, or new progress/angle on it, emit `updates` with + action=update for that id - never open it as a new topic. + Example: carry_forward_topics has "explainability evaluation of memory + search" and the material says the evaluation dataset is now ready -> + update that topic; do NOT open "run the benchmark" as a new topic. Open + a new entry only for matters no carry_forward topic covers. + - Quote YAML strings containing punctuation such as `:`; write paths as + block lists so `{daily_dir}//...` stays parseable. + + ## Output format + + Return only one YAML fenced block with this exact shape: + ```yaml + follow_ups: + - title: + reason: + confidence: 0.7 + paths: [, ...] + [extends]extends: + [extends] - title: + [extends] reason: + [extends] confidence: 0.5 + [extends] paths: [, ...] + updates: + - id: + action: keep|update|resolve + evidence: + reason: + confidence: + ``` + +extract_system_prompt_zh: | + 你是个人记忆 workspace 的 proactive 发现 agent。你阅读近期变化的对话笔记 + ，维护一小组用户兴趣主题。你绝不联网、 + 绝不写文件、绝不自行收集资料。 + + workspace_dir: {workspace_dir} + + ## 分支 A - follow_ups（未决事项 / open loop） + + 找出用户可能仍想关闭的未决事项： + - 提出过但没有得到回答的问题； + - 开始过但被中断或搁置的任务； + - 提到过但没有后续动作的承诺或计划。 + 每条作为一个 `follow_ups` 条目输出。 + + [extends]## 分支 B - extends（兴趣边界扩展） + [extends] + [extends]推断用户尚未关注、但与其近期工作大概率相关的主题，依据是对话笔记 + [extends]。它们用于指引未来的知识源扩展；你只描述主题，不收集任何资料。 + [extends]每条作为一个 `extends` 条目输出。 + + ## updates（carry-forward 主题处置） + + 对 carry_forward_topics 中的每个主题，恰好选择一个动作： + - `keep`：仍然 open，但今天没有新证据； + - `update`：今天出现新证据；刷新 evidence 与 confidence（输出按 rubric + 重新打分的 confidence 值）； + - `resolve`：对话显示事项已解决或完成。 + 必须原样回显给定的 `id`。禁止伪造 id，禁止对不在 carry_forward_topics + 中的主题输出 update。 + 若 carry_forward_topics 中有两个主题实为同一件事的不同表述，保留证据较强 + 的一个，对另一个输出 `resolve`（reason 注明 merged into <保留的 id>）。 + 仅当新证据出自本轮 changed_paths 列表中的文件时才输出 action=update； + 否则输出 keep（引用其他文件的 update 会在下游被拒绝并降级为 keep）。 + + ## confidence 评分规则（固化，禁止自由打分） + + - 0.9：有明确未完成动作且带时间/承诺； + - 0.7：用户显式表达意图但无明确动作； + - 0.5：推断性关联（多条弱信号）； + - 0.3：弱联想； + - 低于 0.3：不要输出该主题。 + + ## 硬约束 + + - 空列表是正常且预期的输出。不得为凑数编造主题，不得把一次性弱提及 + 拔高为主题。 + - 每个主题必须能追溯到本轮素材：`paths` 只能使用 changed_paths 中的值 + （路径形如 `{daily_dir}//...`），越界条目会被丢弃。 + - 不要复述 carry_forward_topics 中已 open 的主题；要改变其状态请用 + `updates`。 + - 同一件事规则：若新发现其实只是某个 carry_forward 主题的换种说法， + 或是它的新进展/新角度，必须对该 id 输出 action=update 的 `updates` + 条目，禁止把它当作新主题另开条目。例：carry_forward_topics 已有 + "记忆检索的可解释性评估"，素材提到评测数据集已就绪 -> 更新该主题， + 不得新开"运行 benchmark"主题。仅当 carry_forward 主题均未覆盖时， + 才可新开主题。 + - 包含 `:` 等标点的 YAML 字符串请加引号；paths 使用 block list， + 避免 `{daily_dir}//...` 解析失败。 + + ## 输出格式 + + 只返回一个 YAML fenced block，结构必须严格如下： + ```yaml + follow_ups: + - title: <具体的未决事项> + reason: <为什么仍未关闭> + confidence: 0.7 + paths: [<素材路径>, ...] + [extends]extends: + [extends] - title: <相关但未被关注的主题> + [extends] reason: <为什么重要，基于素材> + [extends] confidence: 0.5 + [extends] paths: [<素材路径>, ...] + updates: + - id: <回显的 carry-forward id> + action: keep|update|resolve + evidence: <路径或 路径#锚点> + reason: <简短理由> + confidence: <按 rubric 重新打分；action=update 时给出，其余可省略> + ``` + +extract_user_message: | + date: {date} + + changed_paths (the ONLY allowed values for `paths`): + {changed_paths_json} + + carry_forward_topics (already open; only act on them via updates): + {carry_forward_json} + + user_profile (background knowledge about the user's identity, preferences + and constraints; use it ONLY to judge interest relevance and confidence - + it is NOT material, never cite it in paths/evidence): + {profile_block} + + # Material + + {material_blob} + + Extract follow_ups[extends] / extends / updates per the contract. Remember: + empty lists are fine; never invent topics. + +extract_user_message_zh: | + 日期：{date} + + changed_paths（`paths` 唯一允许的取值）： + {changed_paths_json} + + carry_forward_topics（已 open 的主题；只能通过 updates 处置）： + {carry_forward_json} + + 用户画像（用户身份、偏好与约束的背景知识；仅用于判断兴趣相关度与 + confidence——它不是素材，禁止引用进 paths/evidence）： + {profile_block} + + # 本轮素材 + + {material_blob} + + 按契约抽取 follow_ups[extends] / extends / updates。记住：空列表是正常 + 输出，绝不编造主题。 diff --git a/reme/steps/evolve/proactive/finish.py b/reme/steps/evolve/proactive/finish.py new file mode 100644 index 00000000..6f63fb2d --- /dev/null +++ b/reme/steps/evolve/proactive/finish.py @@ -0,0 +1,79 @@ +"""Proactive finish step: checkpoint only the proactive catalog (F3).""" + +from ...base_step import BaseStep +from .._evolve import passthrough_response +from ....components import R +from ....schema import FileNode, ProactiveState +from ..dream.utils import workspace_dir + + +@R.register("proactive_finish_step") +class ProactiveFinishStep(BaseStep): + """Upsert this round's changed material into the proactive catalog. + + interests.yaml is deliberately NOT checkpointed (v5 R6): the catalog only + serves change-detection watermarking and interests.yaml is excluded from + the material set (INV-11), so no consumer would ever read it back. + Never calls ``refresh_day_index`` and never touches the dream catalog + (INV-2). When the short-circuit flag is set, no checkpoint happens so the + same material stays "changed" for the next round (INV-7). + """ + + def __init__(self, persist: bool = True, skip_key: str = "proactive_skip", **kwargs): + super().__init__(**kwargs) + self.persist = persist + self.skip_key = skip_key + + async def execute(self): + assert self.context is not None + if self.context.get(self.skip_key): + return passthrough_response(self, self.skip_key) + if self.file_catalog is None: + raise RuntimeError("proactive_finish_step requires file_catalog") + raw_state = self.context.get("proactive") + if not raw_state: + self.context.response.success = True + self.context.response.answer = "Skipped finish: no proactive extract state in context" + return self.context.response + state = ProactiveState.model_validate(raw_state) + if state.early_exit: + self.context.response.success = True + self.context.response.answer = f"Skipped finish: {state.early_exit}" + return self.context.response + ws = workspace_dir(self) + + checkpoint = [rel for rel in state.changed_paths if (ws / rel).is_file()] + snapshot = state.changed_mtimes + nodes: list[FileNode] = [] + deferred = 0 + for rel in dict.fromkeys(checkpoint): + try: + mtime = (ws / rel).stat().st_mtime + except OSError: + continue + if rel in snapshot and mtime != snapshot[rel]: + # Modified while extract/plan/agenda were running: the new + # content never reached this round's prompt, so leave the path + # un-checkpointed for the next round (audit item 3). + deferred += 1 + continue + nodes.append(FileNode(path=rel, st_mtime=mtime)) + self.logger.info( + f"[{self.name}] start checkpoint={len(nodes)} deferred={deferred} persist={self.persist}", + ) + if nodes: + await self.file_catalog.upsert(nodes) + if self.persist and nodes: + await self.file_catalog.dump() + + state.checkpoint_paths = [n.path for n in nodes] + data = state.model_dump() + self.context["proactive"] = data + self.context.response.metadata["proactive"] = data + self.context.response.success = True + answer = f"Proactive finished: checkpointed {len(nodes)} path(s)" + if deferred: + answer += f", deferred {deferred} path(s) modified during the round" + self.context.response.answer = answer + self.logger.info(f"[{self.name}] finish checkpointed={len(nodes)}") + return self.context.response diff --git a/reme/steps/evolve/proactive/plan.py b/reme/steps/evolve/proactive/plan.py new file mode 100644 index 00000000..e29b3af3 --- /dev/null +++ b/reme/steps/evolve/proactive/plan.py @@ -0,0 +1,246 @@ +"""Proactive plan step: expand push candidates into scenario cards (F2.5). + +Runs between the topics and agenda steps. Only topics that qualify for today's +push (``first_seen == today`` and ``confidence >= min_push_confidence``, +computed by the topics step into ``state.push_candidates``) are expanded, so +the LLM cost stays proportional to what will actually be surfaced. One batched +LLM call per round; on any LLM failure every selected candidate still gets a +deterministic fallback card so the agenda step never sees an empty input. + +Card contract: ``scenario_type`` (resume_task | answer_pending | +explore_interest | prepare_upcoming), ``opener`` (a casual, friend-like +conversation opener that already names the minimal next action - never a +"based on your records" notification tone), ``next_action``, ``preconditions`` +and ``delivery`` (in_conversation | notification | agenda_item). +``linked_memory`` is derived from the topic's paths/evidence by code, never by +the LLM. +""" + +import asyncio +import json +import time + +from ...base_step import BaseStep +from .._evolve import agent_reply_result_text, passthrough_response +from ....components import R +from ....schema import ProactiveState +from ..dream.utils import workspace_dir +from .utils import load_personal_profile_block, parse_plan_reply, resolve_agent_wrapper + +SCENARIO_TYPES = ("resume_task", "answer_pending", "explore_interest", "prepare_upcoming") +DELIVERY_MODES = ("in_conversation", "notification", "agenda_item") +DEFAULT_SCENARIO_BY_KIND = {"follow_up": "resume_task", "interest_extend": "explore_interest"} +MAX_OPENER_CHARS = 300 +MAX_NEXT_ACTION_CHARS = 200 +MAX_PRECONDITIONS = 5 +MAX_PRECONDITION_CHARS = 80 + + +def linked_memory(candidate: dict) -> list[str]: + """Workspace paths related to a topic, derived from paths + evidence.""" + out: list[str] = [] + raw = list(candidate.get("paths") or []) + evidence = str(candidate.get("evidence") or "").strip() + if evidence: + raw.append(evidence) + for path in raw: + rel = str(path or "").strip().split("#", 1)[0] + if rel and rel not in out: + out.append(rel) + return out + + +def fallback_card(candidate: dict) -> dict: + """Deterministic card used when the LLM is absent or its output is unusable.""" + kind = str(candidate.get("kind") or "interest_extend") + title = str(candidate.get("title") or "") + scenario = DEFAULT_SCENARIO_BY_KIND.get(kind, "explore_interest") + if kind == "follow_up": + opener = f"上次聊到「{title}」，好像还没收尾——要不要趁现在花几分钟往前推一步？" + next_action = "先回顾相关记录，确认卡点，然后给出下一步动作" + else: + opener = f"感觉你最近可能会想看看「{title}」，有空的话可以先从相关材料扫一眼。" + next_action = "快速浏览相关材料，判断值不值得深入" + memory = linked_memory(candidate) + if memory: + next_action = f"{next_action}（材料：{memory[0]}）" + return { + "scenario_type": scenario, + "opener": opener, + "next_action": next_action, + "preconditions": [], + "delivery": "in_conversation", + } + + +def _clean_card(raw: dict, candidate: dict) -> dict: + """Validate one LLM card; per-field fallback keeps the card always usable.""" + base = fallback_card(candidate) + scenario = str(raw.get("scenario_type") or "").strip() + if scenario not in SCENARIO_TYPES: + scenario = base["scenario_type"] + opener = str(raw.get("opener") or "").strip()[:MAX_OPENER_CHARS] + if not opener: + opener = base["opener"] + next_action = str(raw.get("next_action") or "").strip()[:MAX_NEXT_ACTION_CHARS] + if not next_action: + next_action = base["next_action"] + preconditions = raw.get("preconditions") + if not isinstance(preconditions, list): + preconditions = [] + preconditions = [str(item).strip()[:MAX_PRECONDITION_CHARS] for item in preconditions if str(item).strip()] + delivery = str(raw.get("delivery") or "").strip() + if delivery not in DELIVERY_MODES: + delivery = base["delivery"] + return { + "scenario_type": scenario, + "opener": opener, + "next_action": next_action, + "preconditions": preconditions[:MAX_PRECONDITIONS], + "delivery": delivery, + } + + +@R.register("proactive_plan_step") +class ProactivePlanStep(BaseStep): + """Expand today's push candidates into scenario cards. + + Zero LLM calls when there are no push candidates. At most + ``max_plan_topics`` candidates are expanded; the overflow stays card-less + and the agenda step suppresses it with an explicit reason. + """ + + def __init__( + self, + max_plan_topics: int = 6, + llm_timeout_seconds: float = 120, + profile_max_chars: int = 800, + skip_key: str = "proactive_skip", + **kwargs, + ): + super().__init__(**kwargs) + self.max_plan_topics = max(int(max_plan_topics), 1) + self.llm_timeout_seconds = float(llm_timeout_seconds) + self.profile_max_chars = max(int(profile_max_chars), 0) + self.skip_key = skip_key + + async def execute(self): + assert self.context is not None + if self.context.get(self.skip_key): + return passthrough_response(self, self.skip_key) + started = time.monotonic() + raw_state = self.context.get("proactive") + if not raw_state: + self.context.response.success = True + self.context.response.answer = "Skipped plan: no proactive extract state in context" + return self.context.response + state = ProactiveState.model_validate(raw_state) + if state.early_exit: + self.context.response.success = True + self.context.response.answer = f"Skipped plan: {state.early_exit}" + return self.context.response + + candidates = [c for c in state.push_candidates if isinstance(c, dict) and c.get("id")] + if not candidates: + state.scenario_cards = [] + self._store(state) + self.context.response.success = True + self.context.response.answer = "Plan: no push candidates, 0 cards, 0 LLM calls" + self.logger.info(f"[{self.name}] no push candidates; skipping") + return self.context.response + + selected = candidates[: self.max_plan_topics] + self.logger.info( + f"[{self.name}] start candidates={len(candidates)} selected={len(selected)} " + f"max_plan_topics={self.max_plan_topics}", + ) + cards_by_id = await self._llm_cards(state, selected) + + cards: list[dict] = [] + fallbacks = 0 + for candidate in selected: + cid = str(candidate.get("id")) + raw_card = cards_by_id.get(cid) + if raw_card is None: + fallbacks += 1 + card = _clean_card(raw_card, candidate) if raw_card is not None else fallback_card(candidate) + card = { + "topic_id": cid, + "title": str(candidate.get("title") or ""), + "kind": str(candidate.get("kind") or "interest_extend"), + **card, + "linked_memory": linked_memory(candidate), + } + cards.append(card) + + state.scenario_cards = cards + state.duration_ms = state.duration_ms + int((time.monotonic() - started) * 1000) + self._store(state) + answer = f"Plan: {len(cards)} scenario card(s), fallback={fallbacks}, llm_calls={state.plan_llm_calls}" + self.context.response.success = True + self.context.response.answer = answer + self.logger.info(f"[{self.name}] finish {answer}") + return self.context.response + + async def _llm_cards(self, state: ProactiveState, selected: list[dict]) -> dict[str, dict]: + """One batched LLM call; returns candidate-id -> raw card dict.""" + wrapper = resolve_agent_wrapper(self) + if wrapper is None: + self.logger.warning(f"[{self.name}] no agent_wrapper available; using fallback cards") + return {} + selected_ids = {str(c.get("id")) for c in selected} + ws = workspace_dir(self) + user_message = self.prompt_format( + "plan_user_message", + profile_block=self._profile_block(ws), + candidates_json=json.dumps( + [ + { + "id": c.get("id"), + "title": c.get("title"), + "kind": c.get("kind"), + "reason": c.get("reason"), + "confidence": c.get("confidence"), + "first_seen": c.get("first_seen"), + "evidence": c.get("evidence"), + } + for c in selected + ], + ensure_ascii=False, + ), + ) + system_prompt = self.prompt_format("plan_system_prompt") + state.plan_llm_calls += 1 + try: + result = await asyncio.wait_for( + wrapper.reply(user_message, system_prompt=system_prompt), + timeout=self.llm_timeout_seconds, + ) + except asyncio.TimeoutError: + self.logger.warning(f"[{self.name}] LLM reply timed out after {self.llm_timeout_seconds}s") + return {} + except Exception as e: # noqa: BLE001 - network/provider errors degrade to fallback cards + self.logger.warning( + f"[{self.name}] LLM reply failed ({type(e).__name__}: {e}); using fallback cards", + ) + return {} + raw = agent_reply_result_text(result) + cards_by_id: dict[str, dict] = {} + for card in parse_plan_reply(raw): + cid = str(card.get("topic_id") or "").strip() + if cid in selected_ids and cid not in cards_by_id: + cards_by_id[cid] = card + if not cards_by_id: + self.logger.warning(f"[{self.name}] plan reply unusable; raw={raw[:200]!r}") + return cards_by_id + + def _profile_block(self, ws) -> str: + """Digest-personal profile sketch so openers respect user preferences.""" + digest_dir = str(self.config_value("digest_dir")) + return load_personal_profile_block(ws, digest_dir, self.profile_max_chars) + + def _store(self, state: ProactiveState) -> None: + assert self.context is not None + data = state.model_dump() + self.context["proactive"] = data + self.context.response.metadata["proactive"] = data diff --git a/reme/steps/evolve/proactive/plan.yaml b/reme/steps/evolve/proactive/plan.yaml new file mode 100644 index 00000000..5e416141 --- /dev/null +++ b/reme/steps/evolve/proactive/plan.yaml @@ -0,0 +1,108 @@ +plan_system_prompt: | + You are the conversation-opportunity designer of a personal memory system. + You receive the proactive topics selected for today's push. For EACH topic + produce one scenario card so an assistant can raise it naturally in the next + conversation - and guide the user toward a concrete next step, not merely + inform them about the topic. + + ## Fields per card + + - `topic_id`: echo the given id unchanged. + - `scenario_type`: exactly one of + - `resume_task`: a task that was started but interrupted/postponed; + - `answer_pending`: a question that was asked but never answered; + - `explore_interest`: an interest the user has not dived into yet; + - `prepare_upcoming`: something with a future time anchor to prepare for. + follow_up topics become resume_task / answer_pending / prepare_upcoming; + interest_extend topics become explore_interest / prepare_upcoming. Pick by + semantics; use a time anchor whenever the topic carries one. + - `opener`: one casual spoken sentence, like a friend happening to mention + it ("by the way..."). STRICTLY forbidden: notification tone such as + "based on your previous records", "the system reminds you", "as you + mentioned on ". The opener must already point at the minimal + executable next action, so it invites doing, not just knowing. + - `next_action`: the smallest concrete step the user can start right now. + - `preconditions`: list of things that must hold first; empty list is fine. + - `delivery`: `in_conversation` (weave into talk), `notification` + (standalone nudge) or `agenda_item` (belongs on today's agenda). + + ## Hard rules + + - Exactly one card per input topic, no extras. + - Ground everything in the given reason/evidence; never invent facts. + - Quote YAML strings containing punctuation such as `:`. + + ## Output format + + Return only one YAML fenced block: + ```yaml + cards: + - topic_id: + scenario_type: resume_task|answer_pending|explore_interest|prepare_upcoming + opener: + next_action: + preconditions: [, ...] + delivery: in_conversation|notification|agenda_item + ``` + +plan_system_prompt_zh: | + 你是个人记忆系统的"对话机会设计师"。输入是今天被选中推送的 proactive + 主题。为每个主题产出一张场景卡，让助手能在下一次对话里自然地把话题提起 + ——并引导用户迈出具体下一步，而不只是告知话题存在。 + + ## 每张卡的字段 + + - `topic_id`：原样回显给定 id。 + - `scenario_type`：四选一 + - `resume_task`：开始过但被中断/搁置的任务； + - `answer_pending`：提出过但没有得到回答的问题； + - `explore_interest`：用户尚未深入的兴趣方向； + - `prepare_upcoming`：带未来时间点、需要提前准备的事项。 + follow_up 主题对应 resume_task / answer_pending / prepare_upcoming； + interest_extend 主题对应 explore_interest / prepare_upcoming。按语义选择； + 主题带时间线索时优先 prepare_upcoming。 + - `opener`：一句口语化的开场，像朋友随口提起（"对了，上次那个……"）。 + 严禁通知腔："根据您之前的记录"、"系统提醒您"、"您在某天提到过"等。 + 开场必须顺势给出最小可执行的下一步，让人愿意动手，而不只是知道。 + - `next_action`：用户现在就能开始的最小具体动作。 + - `preconditions`：需要先满足的条件列表，可以为空。 + - `delivery`：`in_conversation`（对话中自然带出）/ `notification` + （单独提醒）/ `agenda_item`（列入当日议程）。 + + ## 硬约束 + + - 每个输入主题恰好一张卡，不得多给。 + - 所有内容必须基于给定的 reason/evidence，禁止编造事实。 + - 包含 `:` 等标点的 YAML 字符串请加引号。 + + ## 输出格式 + + 只返回一个 YAML fenced block： + ```yaml + cards: + - topic_id: <回显的 id> + scenario_type: resume_task|answer_pending|explore_interest|prepare_upcoming + opener: <一句口语化开场，顺势给出下一步动作> + next_action: <最小可执行步骤> + preconditions: [<条件>, ...] + delivery: in_conversation|notification|agenda_item + ``` + +plan_user_message: | + user_profile (background: let it tune the opener's language and tone; it is + NOT material, never cite it): + {profile_block} + + push_candidates (one scenario card per topic): + {candidates_json} + + Emit the `cards` block per the contract. + +plan_user_message_zh: | + 用户画像（背景信息：用于把握开场白的语言与语气；它不是素材，禁止引用）： + {profile_block} + + 推送候选主题（每个主题一张场景卡）： + {candidates_json} + + 按契约输出 `cards` block。 diff --git a/reme/steps/evolve/proactive/proactive.py b/reme/steps/evolve/proactive/proactive.py new file mode 100644 index 00000000..b84f89a3 --- /dev/null +++ b/reme/steps/evolve/proactive/proactive.py @@ -0,0 +1,204 @@ +"""Read interests.yaml for proactive consumption (F5). + +Migrated from ``dream/proactive.py`` (INV-9 mechanical migration) and extended +with ``min_confidence`` filtering (default 0.4, safely below the 0.5 confidence +fallback) and ``horizon_days``: horizon=1 keeps the legacy single-day file +read (v1 compatible); horizon>1 reads the truth source directly, filtering by +``last_evidence_at`` recency (v5 R4 - the truth source already carries the +cross-day merge, so the reader no longer re-scans exposure products). +""" + +import datetime as dt + +import yaml + +from ...base_step import BaseStep +from ....components import R +from ....schema import ProactiveResult +from ..dream.utils import today, workspace_dir +from .utils import ( + dump_topic, + load_yaml_topics, + load_state, + parse_interests_topics, + sort_topics, + state_file_path, + topic_id, +) + + +@R.register("proactive_step") +class ProactiveStep(BaseStep): + """Read ``daily//interests.yaml`` (optionally merged over a horizon). + + - v1 files (nightly) are read as ``push=true`` and never rewritten; + - ``push=false`` days contribute nothing; + - resolved ids (truth source registry) are suppressed; + - ``horizon_days>1`` reads the truth source and filters by evidence recency. + """ + + def __init__(self, include_content: bool = True, horizon_days: int = 1, min_confidence: float = 0.4, **kwargs): + super().__init__(**kwargs) + self.include_content = include_content + self.horizon_days = max(int(horizon_days), 1) + self.min_confidence = float(min_confidence) + + async def execute(self): + assert self.context is not None + day = today(self, str(self.context.get("date", "") or "")) + include_content = bool(self.context.get("include_content", self.include_content)) + horizon = int(self.context.get("horizon_days", self.horizon_days) or self.horizon_days) + horizon = max(horizon, 1) + raw_min_confidence = self.context.get("min_confidence", self.min_confidence) + try: + min_confidence = float(raw_min_confidence) + except (TypeError, ValueError): + min_confidence = self.min_confidence + daily = self.config_value("daily_dir") + ws = workspace_dir(self) + default_rel = f"{daily}/{day}/interests.yaml" + result = ProactiveResult(date=day, path=default_rel) + self.logger.info( + f"[{self.name}] start date={day} path={default_rel} include_content={include_content} " + f"horizon_days={horizon} min_confidence={min_confidence}", + ) + + if horizon == 1: + outcome = self._read_single(ws, daily, day, include_content, min_confidence, result) + if outcome is not None: + return outcome + + return self._read_horizon(ws, daily, day, horizon, include_content, min_confidence, result) + + # ------------------------------------------------------------------ + # Single-day read (legacy-compatible path) + # ------------------------------------------------------------------ + + def _read_single(self, ws, daily, day, include_content, min_confidence, result: ProactiveResult): + rel_path = f"{daily}/{day}/interests.yaml" + abs_path = ws / daily / day / "interests.yaml" + if not abs_path.is_file(): + result.skipped, result.summary = True, f"Skipped: interests file not found at {rel_path}" + self.logger.info(f"[{self.name}] skip missing path={rel_path}") + return self._finish(True, result, include_content=include_content) + try: + raw_text = abs_path.read_text(encoding="utf-8") + # The reader stays strictly read-only: a corrupt file is never + # moved or renamed here; the parse failure propagates to the + # outer handler and surfaces as success=False so callers can see + # it instead of a silent empty read (audit item 2). + data = yaml.safe_load(raw_text) + if not isinstance(data, dict): + raise ValueError("interests.yaml is not a mapping") + topics, is_v1, push = parse_interests_topics(data, day) + if push is False: + result.skipped = True + result.summary = f"Skipped: interests file at {rel_path} has push=false" + result.push = False + self.logger.info(f"[{self.name}] skip push=false path={rel_path}") + return self._finish(True, result, include_content=include_content) + + result.push = True + result.content = raw_text if include_content else "" + if is_v1: + # Legacy shape (title/reason/evidence/keywords/paths) for v1 files; + # v1 topics carry no confidence and fall back to 0.5 for filtering. + resolved_ids = self._resolved_ids(ws, daily) + legacy_topics = load_yaml_topics(abs_path) + kept = [ + {k: v for k, v in t.items() if k != "keywords"} + for t in legacy_topics + if topic_id(str(t.get("title") or "")) not in resolved_ids + ] + if min_confidence > 0.5 + 1e-9: # v1 topics carry no confidence; fallback is 0.5 (F1.1) + kept = [] + result.topics = kept + else: + result.generated_at = str(data.get("generated_at") or "") + resolved_ids = self._resolved_ids(ws, daily) + kept = [ + dump_topic(t) + for t in sort_topics(topics) + if t.id not in resolved_ids and t.confidence >= min_confidence - 1e-9 + ] + result.topics = kept + agenda_raw = data.get("agenda") + if isinstance(agenda_raw, list): + kept_ids = {str(t.get("id") or "") for t in kept} + result.agenda = [ + item + for item in agenda_raw + if isinstance(item, dict) and str(item.get("topic_id") or "") in kept_ids + ] + result.summary = f"Read {len(result.topics)} proactive topic(s) from {rel_path}" + self.logger.info(f"[{self.name}] read done path={rel_path} topics={len(result.topics)}") + return self._finish(True, result, include_content=include_content) + except Exception as e: # noqa: BLE001 + result.error, result.summary = f"{type(e).__name__}: {e}", "" + self.logger.error(f"[{self.name}] read failed path={rel_path}: {result.error}") + return self._finish(False, result, include_content=include_content) + + # ------------------------------------------------------------------ + # Multi-day merge (horizon_days > 1, or fallback target) + # ------------------------------------------------------------------ + + def _read_horizon(self, ws, daily, day, horizon, include_content, min_confidence, result: ProactiveResult): + """Truth-source view (v5 R4): open topics with evidence inside the horizon. + + The truth source already carries the cross-day merge (carry-forward), + so the reader no longer re-scans N days of exposure products. + """ + state_file, _needs_bootstrap = load_state(ws, daily) + if include_content: + try: + result.content = state_file_path(ws, daily).read_text(encoding="utf-8") + except OSError: + result.content = "" + if not state_file.open_topics: + result.skipped = True + result.summary = f"Skipped: truth source has no open topics (horizon_days={horizon})" + self.logger.info(f"[{self.name}] skip empty truth source horizon={horizon}") + return self._finish(True, result, include_content=include_content) + try: + base = dt.date.fromisoformat(day) + cutoff = (base - dt.timedelta(days=max(horizon - 1, 0))).isoformat() + upper_bound = base.isoformat() + except ValueError: + cutoff = "" + upper_bound = "" + kept = [ + dump_topic(t) + for t in state_file.open_topics + if ( + (not cutoff or cutoff <= str(t.last_evidence_at or "") <= upper_bound) + and t.confidence >= min_confidence - 1e-9 + ) + ] + kept = sort_topics(kept) + result.path = f"{daily}/_proactive.yaml" + result.topics = kept + result.summary = f"Read {len(kept)} proactive topic(s) from the truth source (horizon_days={horizon})" + self.logger.info(f"[{self.name}] truth-source read done horizon={horizon} topics={len(kept)}") + return self._finish(True, result, include_content=include_content) + + def _resolved_ids(self, ws, daily: str) -> set[str]: + state_file, _needs_bootstrap = load_state(ws, daily) + return {str(r.get("id") or "") for r in state_file.resolved if isinstance(r, dict) and r.get("id")} + + def _finish(self, success: bool, result: ProactiveResult, *, include_content: bool): + assert self.context is not None + self.context.response.success = success + if not success: + self.context.response.answer = f"Error: {result.error}" + elif result.skipped: + self.context.response.answer = result.summary + else: + self.context.response.answer = { + "summary": result.summary, + "topics": result.topics, + **({"agenda": result.agenda} if result.agenda else {}), + **({"content": result.content} if include_content else {}), + } + self.context.response.metadata.update(result.model_dump()) + self.logger.info(f"[{self.name}] finish success={success} answer={self.context.response.answer!r}") + return self.context.response diff --git a/reme/steps/evolve/proactive/topics.py b/reme/steps/evolve/proactive/topics.py new file mode 100644 index 00000000..9bf5df75 --- /dev/null +++ b/reme/steps/evolve/proactive/topics.py @@ -0,0 +1,462 @@ +"""Proactive topics step: dedup, truth-source update, derived push, render (F2.4). + +Pure computation step (v5 R2): no LLM calls, no budget, no agent_wrapper. +Semantic dedup keeps only the ``>= known_threshold`` "known" drop; everything +below is kept (loose-not-leaky). The main dedup defense lives upstream in the +extract prompt's same-matter rule plus same-id merging here. + +``known_threshold`` is bound to the embedding model, dimensions AND vector-text +scheme it was calibrated for (v5.2: ``title。reason`` texts, threshold 0.85, +34-pair measurement: DUP band 0.773-0.943 vs KEEP band <=0.772). Cosine +magnitudes are not comparable across models, so a fingerprint mismatch degrades +the gate to exact normalize comparison instead of silently misfiring. Without +any configured embedder the step skips the semantic gate entirely and the +workflow continues on exact comparison (BM25-only deployments). +""" + +import datetime as dt +import re +import time + +from ...base_step import BaseStep +from .._evolve import passthrough_response +from ....components import R +from ....enumeration import ComponentEnum +from ....schema import ProactiveState, ProactiveStateFile, ProactiveTopic +from ....schema.proactive import clamp_confidence +from ..dream.utils import daily_dir, previous_dates, today, workspace_dir +from .utils import ( + current_now, + dump_topic, + interests_path_for, + load_state, + norm_path, + normalize_topic, + parse_interests_topics, + read_interests_data, + render_interests, + save_state, + sort_topics, + trim_state_file, + write_interests_if_changed, +) + + +@R.register("proactive_topics_step") +class ProactiveTopicsStep(BaseStep): + """Filter candidates, update the truth source, and render interests.yaml. + + Without ``as_embedding`` the exact ``normalize_topic`` comparison applies. + With embeddings, ``sim >= known_threshold`` drops as known; below the + threshold candidates are kept. Candidates whose id matches a resolved + tombstone are resurrected (tombstone removed, original ``first_seen`` + kept) instead of being silently suppressed. + """ + + def __init__( + self, + known_threshold: float = 0.85, + min_push_confidence: float = 0.5, + max_topics: int = 10, + dedup_lookback_days: int = 7, + digest_compare_limit: int = 500, + known_threshold_calibrated_for: str = "text-embedding-v4@1024", + skip_key: str = "proactive_skip", + **kwargs, + ): + super().__init__(**kwargs) + self.known_threshold = float(known_threshold) + self.known_threshold_calibrated_for = str(known_threshold_calibrated_for or "") + self.min_push_confidence = float(min_push_confidence) + self.max_topics = max(int(max_topics), 1) + self.dedup_lookback_days = max(int(dedup_lookback_days), 0) + self.digest_compare_limit = max(int(digest_compare_limit), 0) + self.skip_key = skip_key + + # pylint: disable=too-many-statements + async def execute(self): + assert self.context is not None + if self.context.get(self.skip_key): + return passthrough_response(self, self.skip_key) + started = time.monotonic() + if not self.context.get("proactive"): + self.context.response.success = True + self.context.response.answer = "Skipped topics: no proactive extract state in context" + return self.context.response + state = ProactiveState.model_validate(self.context.get("proactive")) + if state.early_exit: + self.context.response.success = True + self.context.response.answer = f"Skipped topics: {state.early_exit}" + return self.context.response + day = state.date or today(self, str(self.context.get("date", "") or "")) + ws = workspace_dir(self) + daily = state.daily_dir or daily_dir(self) + self.logger.info( + f"[{self.name}] start date={day} follow_ups={len(state.follow_ups)} extends={len(state.extends)} " + f"updates={len(state.updates)} carry_forward={state.carry_forward_count}", + ) + + state_file, _needs_bootstrap = load_state(ws, daily) + open_by_id = {t.id: t for t in state_file.open_topics if t.id} + expiry_cutoff = self._expiry_cutoff(day, max(state.carry_forward_days, 1)) + + # 1) Apply updates to the truth source (F2.3). An action=update is only + # legal when its evidence anchors this round's new material (v5.2 hard + # check, same shape as INV-8); otherwise it degrades to keep so stale + # re-reads cannot rewrite evidence or refresh the freshness sort key. + new_material = {norm_path(str(path).split("#", 1)[0]) for path in state.changed_paths} + for update in state.updates: + topic = open_by_id.get(str(update.get("id") or "")) + if topic is None: + continue + action = str(update.get("action") or "keep") + if action == "update": + evidence_raw = str(update.get("evidence") or "").strip() + evidence_path = norm_path(evidence_raw.split("#", 1)[0]) + if evidence_path in new_material: + topic.last_evidence_at = self._evidence_date(evidence_path) or day + topic.evidence = evidence_raw[:120] + if update.get("confidence") is not None: + topic.confidence = clamp_confidence(update.get("confidence")) + state.updates_applied += 1 + else: + self.logger.info( + f"[{self.name}] update for {topic.id} downgraded to keep: evidence " + f"{evidence_path or ''!r} not in this round's new material", + ) + elif action == "resolve": + state_file.open_topics = [t for t in state_file.open_topics if t.id != topic.id] + del open_by_id[topic.id] + state_file.resolved.append( + { + "id": topic.id, + "title": topic.title, + "resolved_at": day, + "first_seen": topic.first_seen, + "evidence": str(update.get("evidence") or "")[:120], + }, + ) + state.updates_resolved += 1 + + # 2) Candidates: same-id merge, tombstone resurrect, dedup the rest. + raw_candidates = list(state.follow_ups) + list(state.extends) + state.candidates_in = len(raw_candidates) + merged: list[dict] = [] + resurrected: list[dict] = [] + fresh: list[dict] = [] + tombstone_by_id = { + str(r.get("id") or ""): r for r in state_file.resolved if isinstance(r, dict) and r.get("id") + } + for candidate in raw_candidates: + cid = str(candidate.get("id") or "") + existing = open_by_id.get(cid) + if existing is not None: + if expiry_cutoff and str(existing.first_seen or "") <= expiry_cutoff: + # Over-age re-mention: trim would prune it this round, so a + # fresh mention restarts the lifetime (unfinished business + # must keep being re-executed, v5.1). + self.logger.info( + f"[{self.name}] over-age topic {cid} re-mentioned; restarting first_seen " + f"({existing.first_seen} -> {day})", + ) + existing.first_seen = day + self._merge_into(existing, candidate, day) + merged.append(candidate) + continue + tombstone = tombstone_by_id.get(cid) + if tombstone is not None: + topic = self._resurrect(tombstone, candidate, day) + state_file.resolved = [r for r in state_file.resolved if r is not tombstone] + del tombstone_by_id[cid] + state_file.open_topics.append(topic) + open_by_id[cid] = topic + resurrected.append(candidate) + self.logger.info(f"[{self.name}] resurrected resolved topic {cid} ({topic.title!r})") + continue + fresh.append(candidate) + + dropped_duplicate = dropped_known = 0 + survivors: list[dict] = [] + seen_normalized: set[str] = set() + if fresh: + comparison = await self._comparison_texts(ws, daily, day, state_file, expiry_cutoff) + embedder = self._resolve_embedding() + if embedder is not None and self.known_threshold_calibrated_for: + fingerprint = self._embedding_fingerprint(embedder) + if fingerprint != self.known_threshold_calibrated_for: + self.logger.warning( + f"[{self.name}] embedding fingerprint {fingerprint or ''!r} != " + f"known_threshold calibration {self.known_threshold_calibrated_for!r}; " + f"cosine is not comparable across models, degrading to exact normalize comparison", + ) + embedder = None + comparison_norms = {normalize_topic(title) for title, _ in comparison} + for candidate in fresh: + normalized = normalize_topic(str(candidate.get("title") or "")) + if normalized in seen_normalized: + dropped_duplicate += 1 + continue + if embedder is None: + if normalized in comparison_norms: + dropped_duplicate += 1 + continue + else: + verdict, matched, similarity = await self._semantic_verdict( + embedder, + candidate, + comparison, + comparison_norms, + ) + if verdict == "known": + dropped_known += 1 + self.logger.info( + f"[{self.name}] candidate {candidate.get('title')!r} dropped as known " + f"(sim={similarity:.3f}, matched={matched!r})", + ) + continue + if verdict == "duplicate": # embedding failure fallback + dropped_duplicate += 1 + continue + seen_normalized.add(normalized) + survivors.append(candidate) + state.dropped_duplicate = dropped_duplicate + state.dropped_known = dropped_known + state.candidates = merged + resurrected + survivors + self.logger.info( + f"[{self.name}] candidates in={state.candidates_in} merged={len(merged)} " + f"resurrected={len(resurrected)} new={len(survivors)} " + f"dropped_duplicate={dropped_duplicate} dropped_known={dropped_known}", + ) + + # 3) Truth source: add new topics, prune, single atomic write (F1.3). + for candidate in survivors: + state_file.open_topics.append(ProactiveTopic.model_validate(candidate)) + trim_state_file(state_file, day, max(state.carry_forward_days, 1)) + await save_state(ws, state_file, daily) + + # 4) Push derived from the cumulative truth source (v5 R1): a topic + # discovered today with sufficient confidence. Monotonic across same-day + # rounds because such topics persist in the truth source once added. + # The candidate list (sorted deterministically) feeds the plan/agenda + # steps; ``push`` is exactly "at least one candidate". + push_candidates = sort_topics( + [t for t in state_file.open_topics if t.first_seen == day and t.confidence >= self.min_push_confidence], + ) + push = bool(push_candidates) + if push: + file_skip_reason = "" + elif merged or resurrected or survivors: + file_skip_reason = "low_confidence" + else: + file_skip_reason = "all_duplicates" + + # 5) Render from the truth source and write idempotently (A4). + topics_out = sort_topics(list(state_file.open_topics))[: self.max_topics] + now_dt = current_now(self) + rendered = render_interests(day, topics_out, push, now_dt) + interests_path = interests_path_for(ws, daily, day) + written = write_interests_if_changed(ws, interests_path, rendered) + rel_path = norm_path(interests_path.relative_to(ws).as_posix()) + + state.topics_out = [dump_topic(t) for t in topics_out] + state.push_candidates = [dump_topic(t) for t in push_candidates] + state.push = push + state.file_skip_reason = file_skip_reason + state.interests_path = rel_path + state.interests_written = written + state.duration_ms = int((time.monotonic() - started) * 1000) + self._store(state) + answer = ( + f"Topics: {len(topics_out)} rendered, push={push}, " + f"skip_reason={file_skip_reason or '-'}, written={written} to {rel_path}" + ) + self.context.response.success = True + self.context.response.answer = answer + self.logger.info(f"[{self.name}] finish {answer}") + return self.context.response + + @staticmethod + def _merge_into(existing: ProactiveTopic, candidate: dict, day: str) -> None: + """Same-id candidate refreshes evidence in place; first_seen is kept.""" + existing.last_evidence_at = day + if candidate.get("reason"): + existing.reason = str(candidate["reason"]) + if candidate.get("evidence"): + existing.evidence = str(candidate["evidence"])[:120] + existing.confidence = clamp_confidence(candidate.get("confidence")) + if candidate.get("paths"): + existing.paths = list(candidate["paths"]) + + @staticmethod + def _resurrect(tombstone: dict, candidate: dict, day: str) -> ProactiveTopic: + """Reopen a resolved topic: original first_seen kept, evidence refreshed. + + The over-age trim still applies to the original ``first_seen``, so a + resurrection only extends a lifetime that has not fully elapsed. + """ + return ProactiveTopic( + id=str(tombstone.get("id") or candidate.get("id") or ""), + title=str(candidate.get("title") or tombstone.get("title") or ""), + reason=str(candidate.get("reason") or ""), + kind=str(candidate.get("kind") or "interest_extend"), + confidence=clamp_confidence(candidate.get("confidence")), + first_seen=str(tombstone.get("first_seen") or day), + last_evidence_at=day, + evidence=str(candidate.get("evidence") or "")[:120], + paths=candidate.get("paths") or [], + ) + + async def _comparison_texts( + self, + ws, + daily: str, + day: str, + state_file: ProactiveStateFile, + expiry_cutoff: str = "", + ) -> list[tuple[str, str]]: + """(title, embed_text) pairs from recent interests, open topics, digest nodes. + + Embed text carries the reason when available (v5.2 calibration: + title+reason separates the DUP/KEEP bands; bare titles do not). + Over-age open topics are excluded (v5.1): they are pruned by trim this + round, so using them as "known" would silently swallow a re-mention of + a matter that is about to restart. + """ + pairs: list[tuple[str, str]] = [] + for previous_day in previous_dates(day, self.dedup_lookback_days): + data = read_interests_data(interests_path_for(ws, daily, previous_day)) + if not data: + continue + topics, _is_v1, _push = parse_interests_topics(data, previous_day) + pairs.extend((t.title, _known_text(t.title, getattr(t, "reason", ""))) for t in topics) + pairs.extend( + (t.title, _known_text(t.title, t.reason)) + for t in state_file.open_topics + if not expiry_cutoff or str(t.first_seen or "") > expiry_cutoff + ) + pairs.extend((title, title) for title in await self._digest_titles()) + return pairs + + async def _digest_titles(self) -> list[str]: + if self.app_context is None or self.digest_compare_limit <= 0: + return [] + catalog = self.app_context.components.get(ComponentEnum.FILE_CATALOG, {}).get("digest") + if catalog is None: + return [] + try: + nodes = await catalog.get_nodes() + except Exception: # noqa: BLE001 + return [] + ordered = sorted(nodes, key=lambda n: float(getattr(n, "st_mtime", 0.0) or 0.0), reverse=True) + return [str(n.path).rsplit("/", 1)[-1].rsplit(".", 1)[0] for n in ordered[: self.digest_compare_limit]] + + def _resolve_embedding(self): + if self.context is not None: + candidate = self.context.get("as_embedding") + if candidate is not None: + return candidate + name = self.kwargs.get("as_embedding", "default") + if self.app_context is None: + return None + return self.app_context.components.get(ComponentEnum.AS_EMBEDDING, {}).get(name) + + async def _semantic_verdict( + self, + embedder, + candidate: dict, + comparison: list[tuple[str, str]], + comparison_norms: set[str], + ) -> tuple[str, str, float]: + """Semantic gate: known (>= known_threshold) | keep; duplicate on embedding failure.""" + candidate_text = _embed_text(candidate) + texts = [candidate_text] + [embed_text for _, embed_text in comparison] + try: + vectors = await embedder(texts) + except Exception as e: # noqa: BLE001 + self.logger.warning(f"[{self.name}] embedding failed, falling back to exact dedup: {e}") + normalized = normalize_topic(str(candidate.get("title") or "")) + return ("duplicate" if normalized in comparison_norms else "keep"), "", 0.0 + if not vectors or len(vectors) != len(texts): + return "keep", "", 0.0 + best, best_idx = 0.0, -1 + for idx, other in enumerate(vectors[1:]): + similarity = _cosine(vectors[0], other) + if similarity > best: + best, best_idx = similarity, idx + matched = comparison[best_idx][0] if 0 <= best_idx < len(comparison) else "" + if best >= self.known_threshold: + return "known", matched, best + return "keep", matched, best + + @staticmethod + def _expiry_cutoff(day: str, carry_forward_days: int) -> str: + """ISO cutoff matching trim_state_file: first_seen <= cutoff is over-age.""" + try: + base = dt.date.fromisoformat(day) + except ValueError: + return "" + return (base - dt.timedelta(days=max(int(carry_forward_days), 0))).isoformat() + + @staticmethod + def _evidence_date(path: str) -> str: + """Date embedded in a daily evidence path; '' when unparseable (v5.2). + + Lets last_evidence_at reflect when the evidence actually happened + (daily/2026-08-12/x.md -> 2026-08-12) instead of always today, so an + update anchored on an older file cannot game the freshness sort key. + """ + match = _EVIDENCE_DATE_RE.search(path or "") + if match: + try: + return dt.date.fromisoformat(match.group(1)).isoformat() + except ValueError: + return "" + return "" + + @staticmethod + def _embedding_fingerprint(embedder) -> str: + """`model@dimensions` of the resolved embedder; '' when not introspectable.""" + model = "" + kwargs = getattr(embedder, "kwargs", None) + if isinstance(kwargs, dict): + model = str(kwargs.get("model") or "") + if not model: + model = str(getattr(getattr(embedder, "model", None), "model", "") or "") + try: + dimensions = int(getattr(embedder, "dimensions", 0) or 0) + except Exception: # noqa: BLE001 - property may raise RuntimeError pre-init + dimensions = 0 + return f"{model}@{dimensions}" if model and dimensions else "" + + def _store(self, state: ProactiveState) -> None: + assert self.context is not None + data = state.model_dump() + self.context["proactive"] = data + self.context.response.metadata["proactive"] = data + + +def _embed_text(candidate: dict) -> str: + """Vector text for a candidate (v5.2 calibration: title+reason).""" + title = str(candidate.get("title") or "") + reason = str(candidate.get("reason") or "").strip() + return f"{title}。{reason}" if reason else title + + +def _known_text(title: str, reason: str) -> str: + """Vector text for a known topic: title+reason when available, else title.""" + reason = str(reason or "").strip() + return f"{title}。{reason}" if reason else title + + +_EVIDENCE_DATE_RE = re.compile(r"(?:^|/)(\d{4}-\d{2}-\d{2})/") + + +def _cosine(a: list[float], b: list[float]) -> float: + if not a or not b or len(a) != len(b): + return 0.0 + dot = sum(x * y for x, y in zip(a, b)) + norm_a = sum(x * x for x in a) ** 0.5 + norm_b = sum(x * x for x in b) ** 0.5 + if norm_a == 0.0 or norm_b == 0.0: + return 0.0 + return dot / (norm_a * norm_b) diff --git a/reme/steps/evolve/proactive/utils.py b/reme/steps/evolve/proactive/utils.py new file mode 100644 index 00000000..a4a5b0ef --- /dev/null +++ b/reme/steps/evolve/proactive/utils.py @@ -0,0 +1,689 @@ +"""Shared proactive helpers: frozen topic identity, truth-source state, rendering. + +Implements the A7 skeleton from PROACTIVE_SPEC.md. ``normalize_topic`` is a +frozen contract (INV-4): any change to it drifts every historical topic id. +""" + +import contextlib +import datetime as dt +import hashlib +import os +import re +import tempfile +import time +import unicodedata +from pathlib import Path + +import yaml + +from ....enumeration import ComponentEnum +from ....schema import ProactiveStateFile, ProactiveTopic +from ....schema.proactive import clamp_confidence +from ....utils import get_logger +from ...file_io._file_io import get_path_lock +from .._evolve import now +from ..dream.utils import clean_paths, recent_dates, scan_day_files + +logger = get_logger(log_to_file=False) + +PROACTIVE_STATE_NAME = "_proactive.yaml" +INTERESTS_NAME = "interests.yaml" +EXTRACT_SECTIONS = ("follow_ups", "extends", "updates") + + +def load_yaml_topics(path: Path, *, strict: bool = False) -> list[dict]: + """Load legacy or current interests YAML topics.""" + if not path.is_file(): + return [] + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + except Exception as exc: + if strict: + raise ValueError(f"Invalid interests YAML at {path}: {exc}") from exc + return [] + if data is None: + if strict: + raise ValueError(f"Invalid interests YAML at {path}: expected an object") + return [] + topics = data.get("topics") if isinstance(data, dict) else None + if not isinstance(topics, list): + if strict: + raise ValueError(f"Invalid interests YAML at {path}: topics must be a list") + return [] + cleaned_topics = [] + for index, topic in enumerate(topics): + if strict: + _validate_topic(topic, path, index) + if isinstance(topic, dict) and (cleaned := _clean_legacy_topic(topic)): + cleaned_topics.append(cleaned) + return cleaned_topics + + +def _validate_topic(topic: object, path: Path, index: int) -> None: + """Reject topic data that would otherwise be silently discarded or coerced.""" + prefix = f"Invalid interests YAML at {path}: topics[{index}]" + if not isinstance(topic, dict): + raise ValueError(f"{prefix} must be an object") + + allowed = {"title", "reason", "evidence", "keywords", "paths"} + if unknown := sorted(set(topic) - allowed): + raise ValueError(f"{prefix} has unknown field(s): {', '.join(str(key) for key in unknown)}") + + for field in ("title", "reason"): + value = topic.get(field) + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{prefix}.{field} must be a non-empty string") + if "evidence" in topic and not isinstance(topic["evidence"], str): + raise ValueError(f"{prefix}.evidence must be a string") + for field in ("keywords", "paths"): + if field not in topic: + continue + values = topic[field] + if not isinstance(values, list) or any(not isinstance(value, str) or not value.strip() for value in values): + raise ValueError(f"{prefix}.{field} must be a list of non-empty strings") + + +def _clean_legacy_topic(raw: dict) -> dict: + """Normalize the v1 topic shape returned by the compatibility reader.""" + title = str(raw.get("title") or "").strip() + reason = str(raw.get("reason") or "").strip() + if not title or not reason: + return {} + keywords = raw.get("keywords") or [] + paths = raw.get("paths") or [] + return { + "title": title, + "reason": reason, + "evidence": str(raw.get("evidence") or "").strip(), + "keywords": ([str(k).strip() for k in keywords if str(k).strip()] if isinstance(keywords, list) else []), + "paths": ([str(p).strip() for p in paths if str(p).strip()] if isinstance(paths, list) else []), + } + + +# --------------------------------------------------------------------------- +# Frozen identity contract (A7 / INV-4) +# --------------------------------------------------------------------------- + + +def normalize_topic(title: str) -> str: + """NFKC -> casefold -> keep only chars whose category starts with L/N. + + Frozen contract (INV-4): removing all whitespace/punctuation means any + modification would drift every historical topic id. + """ + text = unicodedata.normalize("NFKC", title or "").casefold() + return "".join(ch for ch in text if unicodedata.category(ch)[0] in ("L", "N")) + + +def topic_id(title: str) -> str: + """Stable topic identity: ``sha1(normalize_topic(title))[:12]``.""" + return hashlib.sha1(normalize_topic(title).encode("utf-8")).hexdigest()[:12] + + +# --------------------------------------------------------------------------- +# Paths and material set M (F2.0) +# --------------------------------------------------------------------------- + + +def state_file_path(ws: Path, daily: str = "daily") -> Path: + """Truth-source path ``daily/_proactive.yaml``.""" + return ws / daily / PROACTIVE_STATE_NAME + + +def interests_path_for(ws: Path, daily: str, day: str) -> Path: + """Exposure-product path ``daily//interests.yaml``.""" + return ws / daily / day / INTERESTS_NAME + + +def norm_path(rel) -> str: + """Normalize a workspace-relative path (posix, no leading ./).""" + text = str(rel or "").strip().replace("\\", "/") + while text.startswith("./"): + text = text[2:] + return text + + +def scan_material_daily(ws: Path, day: str, daily: str, scan_days: int) -> list[str]: + """M_daily: chunk notes in the scan window, minus day indexes and ``_*`` files (INV-11).""" + out: list[str] = [] + for scan_day in recent_dates(day, scan_days): + day_index = f"{daily}/{scan_day}.md" + for rel in scan_day_files(ws, scan_day, daily): + rel = norm_path(rel) + base = rel.rsplit("/", 1)[-1] + if rel == day_index or base.startswith("_"): + continue + if rel not in out: + out.append(rel) + return sorted(out) + + +# --------------------------------------------------------------------------- +# Truth-source state file daily/_proactive.yaml (F1.3) +# --------------------------------------------------------------------------- + + +def load_state(ws: Path, daily: str = "daily") -> tuple[ProactiveStateFile, bool]: + """Load the truth source; returns ``(state_file, needs_bootstrap)``. + + A missing file means first run (fresh workspace or upgrade) and triggers + the one-time F1.4 bootstrap from interests.yaml history. A corrupt or + invalid file rebuilds empty WITHOUT bootstrap (spec F1.3/A2/A5), as does + an existing file that already carries the ``open_topics`` key (an empty + list is a normal state, not a trigger). + """ + path = state_file_path(ws, daily) + if not path.is_file(): + logger.info(f"proactive state file missing, first-run bootstrap scheduled: {path}") + return ProactiveStateFile(), True + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError("state file is not a mapping") + except Exception as e: # noqa: BLE001 + logger.warning(f"proactive state file corrupt, rebuilding empty: {path} ({e})") + return ProactiveStateFile(), False + needs_bootstrap = "open_topics" not in data + try: + state = ProactiveStateFile.model_validate(data) + except Exception as e: # noqa: BLE001 + logger.warning(f"proactive state file invalid, rebuilding empty: {path} ({e})") + return ProactiveStateFile(), False + return state, needs_bootstrap + + +async def save_state(ws: Path, state_file: ProactiveStateFile, daily: str = "daily") -> None: + """Atomically persist the truth source (path lock + tmp file + os.replace).""" + path = state_file_path(ws, daily) + lock = await get_path_lock(path) + async with lock: + path.parent.mkdir(parents=True, exist_ok=True) + rendered = yaml.safe_dump(state_file.model_dump(), allow_unicode=True, sort_keys=False) + fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(rendered if rendered.endswith("\n") else f"{rendered}\n") + os.replace(tmp, path) + except Exception: + with contextlib.suppress(OSError): + os.unlink(tmp) + raise + + +def _safe_date(text: str) -> dt.date | None: + try: + return dt.date.fromisoformat(str(text or "").strip()) + except ValueError: + return None + + +async def load_carry_forward( + ws: Path, + state_file: ProactiveStateFile, + day: str, + days: int, + top_k: int, + daily: str = "daily", + needs_bootstrap: bool = False, +) -> tuple[list[ProactiveTopic], list[ProactiveTopic]]: + """Return ``(carry_forward_all, carry_forward_prompt)`` sorted per A4 rule 1. + + Bootstraps the truth source from interests.yaml history exactly once on + first run (missing state file) or when an existing file lacks the + ``open_topics`` key (F1.4). Over-age topics are dropped here with a log; + resolved ids are suppressed. + """ + if needs_bootstrap: + state_file.open_topics = _bootstrap_from_history(ws, day, days, daily) + await save_state(ws, state_file, daily) + resolved_ids = {str(r.get("id") or "") for r in state_file.resolved if isinstance(r, dict)} + base = _safe_date(day) + open_topics: list[ProactiveTopic] = [] + expired = 0 + for topic in state_file.open_topics: + if topic.id and topic.id in resolved_ids: + continue + first_seen = _safe_date(topic.first_seen) + # Boundary aligned with trim_state_file/_expiry_cutoff: age >= days is + # over-age everywhere, so a topic never enters the prompt in the same + # round it gets pruned from the truth source (audit item 9). + if base is not None and first_seen is not None and (base - first_seen).days >= int(days): + expired += 1 + continue + open_topics.append(topic) + if expired: + logger.info(f"proactive carry-forward dropped {expired} over-age topic(s) (window={days}d)") + ordered = sort_topics(open_topics) + return ordered, ordered[: max(int(top_k), 0)] + + +def _bootstrap_from_history(ws: Path, day: str, days: int, daily: str) -> list[ProactiveTopic]: + """One-time bootstrap: newest record per id wins, first_seen takes the min (F1.4).""" + if _safe_date(day) is None: + return [] + records: dict[str, ProactiveTopic] = {} + first_seen: dict[str, str] = {} + for file_date in reversed(recent_dates(day, days)): # newest -> oldest + data = read_interests_data(interests_path_for(ws, daily, file_date)) + if not data: + continue + topics, _is_v1, _push = parse_interests_topics(data, file_date) + for topic in topics: + anchor = topic.first_seen or file_date + if topic.id not in first_seen or anchor < first_seen[topic.id]: + first_seen[topic.id] = anchor + records.setdefault(topic.id, topic) + out: list[ProactiveTopic] = [] + for tid, topic in records.items(): + topic.first_seen = first_seen.get(tid) or topic.first_seen or day + out.append(topic) + logger.info(f"proactive bootstrap built {len(out)} open topic(s) from interests.yaml history") + return out + + +def trim_state_file(state_file: ProactiveStateFile, day: str, days: int) -> None: + """Prune budget/exposure/resolved windows and over-age/resolved open topics.""" + base = _safe_date(day) + if base is None: + return + cutoff = (base - dt.timedelta(days=max(int(days), 0))).isoformat() + state_file.resolved = [ + r for r in state_file.resolved if isinstance(r, dict) and str(r.get("resolved_at") or "") > cutoff + ] + resolved_ids = {str(r.get("id") or "") for r in state_file.resolved} + kept: list[ProactiveTopic] = [] + for topic in state_file.open_topics: + if topic.id and topic.id in resolved_ids: + continue + first_seen = _safe_date(topic.first_seen) + if first_seen is not None and first_seen.isoformat() <= cutoff: + continue + kept.append(topic) + state_file.open_topics = kept + + +# --------------------------------------------------------------------------- +# interests.yaml read/render (F1.2 / A2 / A4) +# --------------------------------------------------------------------------- + + +def quarantine_interests(path: Path, error: Exception) -> None: + """Rename a corrupt interests.yaml aside (A2): ``interests.corrupt-.yaml``.""" + stamp = int(time.time()) + corrupt = path.with_name(f"interests.corrupt-{stamp}.yaml") + try: + path.rename(corrupt) + logger.warning(f"quarantined corrupt interests file {path} -> {corrupt.name}: {error}") + except OSError: + logger.warning(f"corrupt interests file {path}: {error}") + + +def read_interests_file(path: Path) -> tuple[str, dict] | None: + """Read interests.yaml returning ``(raw_text, data)``; quarantine corrupt files (A2).""" + if not path.is_file(): + return None + try: + raw_text = path.read_text(encoding="utf-8") + data = yaml.safe_load(raw_text) + if not isinstance(data, dict): + raise ValueError("interests.yaml is not a mapping") + return raw_text, data + except Exception as e: # noqa: BLE001 + quarantine_interests(path, e) + return None + + +def read_interests_data(path: Path) -> dict | None: + """Parse interests.yaml; quarantine corrupt files (A2) and return None.""" + loaded = read_interests_file(path) + return loaded[1] if loaded else None + + +def parse_interests_topics(data: dict, file_date: str) -> tuple[list[ProactiveTopic], bool, bool]: + """Return ``(topics, is_v1, push)`` with A2 fallbacks applied. + + Missing ``first_seen``/``last_evidence_at`` fall back to the file date + (not today); missing ids are derived from the frozen title hash. + """ + is_v1 = data.get("version") is None + push = data.get("push", True) + if not isinstance(push, bool): + push = True + raw_topics = data.get("topics") or [] + topics: list[ProactiveTopic] = [] + for raw in raw_topics if isinstance(raw_topics, list) else []: + if not isinstance(raw, dict): + continue + title = str(raw.get("title") or "").strip() + reason = str(raw.get("reason") or "").strip() + if not title or not reason: + continue + topics.append( + ProactiveTopic( + id=str(raw.get("id") or "").strip() or topic_id(title), + title=title, + reason=reason, + kind=raw.get("kind", "interest_extend"), + confidence=raw.get("confidence", 0.5), + first_seen=str(raw.get("first_seen") or "").strip() or file_date, + last_evidence_at=str(raw.get("last_evidence_at") or "").strip() or file_date, + evidence=str(raw.get("evidence") or "").strip()[:120], + paths=raw.get("paths") or [], + ), + ) + return topics, is_v1, push + + +def sort_topics(topics: list) -> list: + """Order: last_evidence_at desc -> follow_up first -> confidence desc -> id asc. + + Freshness is the primary key (v5 aging fix): stale topics sink below newly + evidenced ones of any kind, so long-lived follow_ups cannot permanently + crowd out new discoveries. + """ + + def get(topic, key): + return getattr(topic, key) if not isinstance(topic, dict) else topic.get(key) + + out = sorted(topics, key=lambda t: str(get(t, "id") or "")) + out.sort(key=lambda t: clamp_confidence(get(t, "confidence")), reverse=True) + out.sort(key=lambda t: 0 if get(t, "kind") == "follow_up" else 1) + out.sort(key=lambda t: str(get(t, "last_evidence_at") or ""), reverse=True) + return out + + +def dump_topic(topic) -> dict: + """Render one topic as an A2-ordered dict for interests.yaml v2.""" + get = (lambda k: getattr(topic, k)) if not isinstance(topic, dict) else topic.get + return { + "id": str(get("id") or ""), + "title": str(get("title") or ""), + "reason": str(get("reason") or ""), + "kind": str(get("kind") or "interest_extend"), + "confidence": clamp_confidence(get("confidence")), + "first_seen": str(get("first_seen") or ""), + "last_evidence_at": str(get("last_evidence_at") or ""), + "evidence": str(get("evidence") or "")[:120], + "paths": [str(p) for p in (get("paths") or [])], + } + + +def render_interests( + day: str, + topics: list, + push: bool, + now_dt: dt.datetime, + agenda: list | None = None, + suppressed: list | None = None, +) -> dict: + """Render the full v2 file content from the truth source (INV-6). + + v5: ``skip_reason`` is no longer persisted (no consumer); it survives as + structured log/metadata on ``ProactiveState.file_skip_reason`` (R7). + The plan/agenda enrichment keys (``agenda``/``suppressed``) are only + present when the agenda step rendered the file; the topics-only renderer + passes ``None`` so its file shape stays unchanged. + """ + rendered = { + "version": 2, + "date": day, + "generated_at": now_dt.isoformat(timespec="seconds"), + "push": bool(push), + "topics": [dump_topic(t) for t in topics], + } + if agenda is not None: + rendered["agenda"] = list(agenda) + if suppressed is not None: + rendered["suppressed"] = list(suppressed) + return rendered + + +def write_interests_if_changed(ws: Path, path: Path, rendered: dict) -> bool: # pylint: disable=unused-argument + """Apply A4 render-write rules (idempotent skip + atomic replace). + + v5 (R1): the "push=false never overwrites nightly v1" special case is + gone; ``push`` is derived from the cumulative truth source, so re-renders + are monotonic and need no guard. + """ + existing: dict | None = None + if path.is_file(): + existing = read_interests_data(path) + if existing is not None: + existing_push = existing.get("push", True) + if not isinstance(existing_push, bool): + existing_push = True + if ( + existing_push == bool(rendered.get("push")) + and existing.get("topics") == rendered.get("topics") + and (existing.get("agenda") or []) == (rendered.get("agenda") or []) + and (existing.get("suppressed") or []) == (rendered.get("suppressed") or []) + ): + return False + path.parent.mkdir(parents=True, exist_ok=True) + payload = yaml.safe_dump(rendered, allow_unicode=True, sort_keys=False) + fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(payload if payload.endswith("\n") else f"{payload}\n") + os.replace(tmp, path) + except Exception: + with contextlib.suppress(OSError): + os.unlink(tmp) + raise + return True + + +# --------------------------------------------------------------------------- +# Candidate cleaning (extract side, A3) +# --------------------------------------------------------------------------- + + +def clean_candidate(raw, allowed_paths: set[str], kind: str, day: str) -> dict: + """Clean one LLM candidate; drop entries whose paths escape M (INV-8).""" + if not isinstance(raw, dict): + return {} + title = str(raw.get("title") or "").strip() + reason = str(raw.get("reason") or "").strip() + paths = clean_paths(raw.get("paths"), allowed_paths) + if not title or not reason or not paths: + return {} + return { + "id": topic_id(title), + "title": title, + "reason": reason, + "kind": kind, + "confidence": clamp_confidence(raw.get("confidence")), + "first_seen": day, + "last_evidence_at": day, + # Derived, not LLM-emitted: the first whitelisted path anchors the + # candidate, so the output contract stays one field smaller. + "evidence": paths[0][:120], + "paths": paths, + } + + +def current_now(step) -> dt.datetime: + """Business-time access per INV-3 (timezone-aware; never datetime.now()).""" + tz = step.app_context.app_config.timezone if step.app_context is not None else None + return now(tz) + + +def parse_extract_reply(text: str) -> dict: + """Parse the A3 fenced YAML/JSON output; fenced blocks take priority. + + A reply only counts as parsed when at least one contract section + (``EXTRACT_SECTIONS``) is present as a list; a non-empty mapping with + misspelled or missing section names is a parse failure, so + ``_extract_with_retry`` retries instead of checkpointing changed files + on output that can never yield topics. + Unlike the dream parser there is no scalar-mapping fallback: proactive + output is sectioned lists, and a partial fallback would corrupt updates. + """ + candidates = [m.group(1).strip() for m in re.finditer(r"```(?:json|ya?ml)?\s*(.*?)```", text, re.S | re.I)] + candidates.append((text or "").strip()) + for raw in candidates: + if not raw: + continue + try: + data = yaml.safe_load(raw) + except yaml.YAMLError: + continue + if isinstance(data, dict) and data and any(isinstance(data.get(key), list) for key in EXTRACT_SECTIONS): + return data + return {} + + +def strip_frontmatter(text: str) -> tuple[dict, str]: + """Split leading ``---`` YAML frontmatter blocks from the body. + + Handles files with several consecutive frontmatter blocks; returns + ``(merged_meta, body)``. Parse failures degrade to empty meta. + """ + meta: dict = {} + body = text or "" + while True: + stripped = body.lstrip() + if not stripped.startswith("---"): + break + rest = stripped[3:] + end = rest.find("\n---") + if end < 0: + break + block = rest[:end] + body = rest[end + 4 :] + try: + data = yaml.safe_load(block) + except yaml.YAMLError: + data = None + if isinstance(data, dict): + for key, value in data.items(): + meta.setdefault(str(key), value) + return meta, body.strip() + + +def _contained_workspace_path(ws: Path, rel_path: str) -> Path | None: + """Resolve ``rel_path`` strictly inside the workspace (audit item 7). + + Rejects absolute paths, home-relative paths and ``..`` traversal so the + profile fallback can never read (and then feed into prompts) files that + live outside the workspace. + """ + candidate = Path(rel_path) + if candidate.is_absolute() or candidate.anchor or rel_path.startswith("~") or ".." in candidate.parts: + return None + workspace = ws.resolve() + resolved = (workspace / candidate).resolve() + try: + resolved.relative_to(workspace) + except ValueError: + return None + return resolved + + +def load_personal_profile_block( + ws: Path, + digest_dir: str, + max_chars: int, + fallback_rel_path: str = "", +) -> str: + """Build the user profile/preference block used to personalize proactive inference. + + Primary source is ``/personal/*.md`` (one file per profile + facet: identity, preferences, constraints). Each file contributes its + frontmatter ``description`` plus a body excerpt under an equal per-file + budget. Falls back to a single profile file (legacy ``profile.md``) when + the personal directory is absent, and to a sentinel when nothing exists. + """ + max_chars = int(max_chars) + if max_chars <= 0: + return "(no user profile)" + personal_dir = ws / digest_dir / "personal" + if personal_dir.is_dir(): + files = sorted(path for path in personal_dir.glob("*.md") if path.is_file()) + if files: + per_file = max(max_chars // len(files), 200) + sections: list[str] = [] + for path in files: + try: + raw = path.read_text(encoding="utf-8") + except OSError: + continue + meta, body = strip_frontmatter(raw) + title = str(meta.get("name") or path.stem) + description = str(meta.get("description") or "").strip() + excerpt = body[:per_file].strip() + section = f"### {title}\n" + if description: + section += f"{description}\n" + if excerpt: + section += excerpt + sections.append(section.rstrip()) + if sections: + return "\n\n".join(sections)[:max_chars] + if fallback_rel_path: + fallback = _contained_workspace_path(ws, fallback_rel_path) + if fallback is not None and fallback.is_file(): + try: + text = fallback.read_text(encoding="utf-8")[:max_chars] + except OSError: + return "(no user profile)" + return text.strip() or "(no user profile)" + return "(no user profile)" + + +def parse_fenced_yaml(text: str) -> dict: + """Parse the first fenced YAML/JSON block (or the bare text) into a dict. + + Shared by the plan/agenda reply parsers; returns ``{}`` when nothing + parses to a mapping. + """ + candidates = [m.group(1).strip() for m in re.finditer(r"```(?:json|ya?ml)?\s*(.*?)```", text, re.S | re.I)] + candidates.append((text or "").strip()) + for raw in candidates: + if not raw: + continue + try: + data = yaml.safe_load(raw) + except yaml.YAMLError: + continue + if isinstance(data, dict): + return data + return {} + + +def parse_plan_reply(text: str) -> list[dict]: + """Parse the plan step reply; requires a ``cards`` list of mappings.""" + data = parse_fenced_yaml(text) + cards = data.get("cards") + if not isinstance(cards, list): + return [] + return [card for card in cards if isinstance(card, dict)] + + +def parse_agenda_reply(text: str) -> tuple[list[dict], list[dict]]: + """Parse the agenda step reply into ``(agenda, suppressed)`` lists. + + ``agenda`` must be present as a list; ``suppressed`` defaults to empty. + """ + data = parse_fenced_yaml(text) + agenda = data.get("agenda") + if not isinstance(agenda, list): + return [], [] + suppressed = data.get("suppressed") + if not isinstance(suppressed, list): + suppressed = [] + return [item for item in agenda if isinstance(item, dict)], [item for item in suppressed if isinstance(item, dict)] + + +def resolve_agent_wrapper(step): + """Return the step's agent_wrapper, falling back to the app default (F4.4).""" + wrapper = step.agent_wrapper + if wrapper is not None: + return wrapper + if step.app_context is not None: + fallback = step.app_context.components.get(ComponentEnum.AGENT_WRAPPER, {}).get("default") + if fallback is not None: + configured = step.kwargs.get("agent_wrapper", "default") + step.logger.warning(f"[{step.name}] agent_wrapper '{configured}' missing; using default") + return fallback + return None diff --git a/skills/reme_memory/SKILL.md b/skills/reme_memory/SKILL.md index 5fde546b..bbb963ec 100644 --- a/skills/reme_memory/SKILL.md +++ b/skills/reme_memory/SKILL.md @@ -53,7 +53,7 @@ the active environment's executable directory is on `PATH`; do not repeatedly re ### 2. Configure optional model credentials Basic file operations, BM25 search, wikilink traversal, and reading existing proactive topics work without model -credentials. `auto_memory`, `auto_resource`, and `auto_dream` require an LLM configuration. +credentials. `auto_memory`, `auto_resource`, `auto_dream`, and proactive refresh require an LLM configuration. When those model-powered jobs are needed, have the user provide valid values through the environment or a `.env` file: @@ -195,22 +195,28 @@ reme auto_resource changes='[{"path":"resource//","change":"ad ## Consolidate and Use Proactive Topics The default service runs background and cron jobs while it remains active. `auto_dream` consolidates daily notes and -resource interpretations into long-term digest memory and generates interest topics. Run it manually when the host owns -the schedule or the user requests consolidation: +resource interpretations into long-term digest memory. Proactive refresh independently generates interest topics. Run +Auto Dream manually when the host owns the schedule or the user requests consolidation: ```bash reme auto_dream date="" ``` +Run proactive refresh once, without exposing its writer job through HTTP or MCP: + +```bash +reme start job=proactive_refresh date="" +``` + Read generated topics with: ```bash -reme proactive date="" +reme proactive_read date="" ``` -`auto_dream` requires LLM credentials. `proactive` reads existing structured topics and works without an LLM call. Pass -`include_content=false` when raw YAML content is unnecessary. The host Agent decides whether and how to mention topics; -ReMe does not independently notify the user or take external action. +`auto_dream` and proactive refresh require LLM credentials. `proactive_read` reads existing structured topics and works +without an LLM call. Pass `include_content=false` when raw YAML content is unnecessary. The host Agent decides whether +and how to mention topics; ReMe does not independently notify the user or take external action. ## Integration Rules diff --git a/tests/integration/test_auto_dream.py b/tests/integration/test_auto_dream.py index e020cf18..6929f046 100644 --- a/tests/integration/test_auto_dream.py +++ b/tests/integration/test_auto_dream.py @@ -1,8 +1,8 @@ -"""Integration test for the 4-step auto_dream job and proactive reader. +"""Integration test for the three-step auto_dream job. Runs against a real LLM. The test seeds a dream workspace, runs ``auto_dream`` for -2026-05-28, verifies digest/interests/catalog effects, then runs ``proactive``. -Agent messages and generated markdown/yaml/jsonl artifacts are copied to +2026-05-28, and verifies digest/catalog effects without producing proactive output. +Agent messages and generated markdown/jsonl artifacts are copied to ``tests/integration/logs/auto_dream_latest/`` for manual inspection. """ @@ -12,8 +12,6 @@ import shutil import sys from pathlib import Path -import yaml - INTEGRATION_DIR = Path(__file__).resolve().parent ARTIFACT_DIR = INTEGRATION_DIR / "logs" / "auto_dream_latest" sys.path.insert(0, str(INTEGRATION_DIR)) @@ -99,8 +97,8 @@ def _file_graph_links(env) -> dict[str, list[dict]]: return out -def test_auto_dream_and_proactive(): - """Run auto_dream end to end, save transcripts/results, then read interests via proactive.""" +def test_auto_dream(): + """Run auto_dream end to end and verify that it does not produce proactive interests.""" async def run(): _reset_artifacts() @@ -121,8 +119,6 @@ def test_auto_dream_and_proactive(): "auto_dream", date=DREAM_DATE, hint="Integration test: preserve SOC2, JWT kid, Redis current_kid, and small-PR facts.", - topic_count=3, - topic_diversity_days=7, ) dumped = await recorder.dump() session_jsonl = sorted((env.workspace_dir / "mem_session" / "agentscope").glob("*.jsonl")) @@ -142,7 +138,7 @@ def test_auto_dream_and_proactive(): interests = env.workspace_dir / "daily" / DREAM_DATE / "interests.yaml" catalog = env.workspace_dir / "metadata" / "file_catalog" / "dream.jsonl.zst" assert changed_note.is_file(), f"changed note missing: {changed_note}" - assert interests.is_file(), f"interests.yaml missing: {interests}" + assert not interests.exists(), f"auto_dream unexpectedly wrote interests.yaml: {interests}" assert catalog.is_file(), f"dream catalog missing: {catalog}" after_digest = _all_digest_text(env) @@ -195,16 +191,6 @@ def test_auto_dream_and_proactive(): "no digest↔digest wikilink found in integrated target markdown\n" f"targets: {target_paths}" ) - interests_text = _print_text_file("interests.yaml", interests) - interests_data = yaml.safe_load(interests_text) or {} - topics = interests_data.get("topics") or [] - assert isinstance(topics, list) and topics, f"no topics in interests.yaml\n{interests_text}" - - proactive = await app.run_job("proactive", date=DREAM_DATE, include_content=True) - assert proactive.success is True, f"proactive failed: {proactive.answer!r}" - assert proactive.metadata.get("path") == f"daily/{DREAM_DATE}/interests.yaml" - assert proactive.metadata.get("topics"), f"proactive returned no topics: {proactive.metadata!r}" - if day_index.is_file(): _print_text_file("day_index.md", day_index) else: @@ -231,5 +217,5 @@ def test_auto_dream_and_proactive(): if __name__ == "__main__": print("=== auto_dream integration test ===") - test_auto_dream_and_proactive() + test_auto_dream() print("\nIntegration test passed!") diff --git a/tests/integration/test_reme_e2e.py b/tests/integration/test_reme_e2e.py index 6afacc16..a7198c3e 100644 --- a/tests/integration/test_reme_e2e.py +++ b/tests/integration/test_reme_e2e.py @@ -145,7 +145,6 @@ async def _run_loop(env, reme) -> None: "auto_dream", date=today, hint="Integration e2e: preserve Project Meridian CRDT/Yjs/WebTransport facts.", - topic_count=3, ) assert dream.success is True, f"auto_dream failed: {dream.answer!r}\n{dream.metadata!r}" dmeta = (dream.metadata or {}).get("dream") or {} @@ -182,14 +181,13 @@ async def _run_loop(env, reme) -> None: "search recalled none of the seeded facts — the provision->" "consolidate->index->search loop is broken" ) - # ---- 5. proactive: read the interests surfaced by the dream -- - proactive = await reme.run_job("proactive", date=today, include_content=True) + # ---- 5. proactive: dream does not produce proactive interests -- + proactive = await reme.run_job("proactive_read", date=today, include_content=True) assert proactive.success is True, f"proactive failed: {proactive.answer!r}" pmeta = proactive.metadata or {} assert pmeta.get("path") == f"daily/{today}/interests.yaml", f"unexpected interests path: {pmeta!r}" - topics = pmeta.get("topics") or [] - print(f"\n[5/5 proactive] topics: {topics}") - assert topics, f"proactive surfaced no interest topics: {pmeta!r}" + assert pmeta.get("skipped") is True, f"proactive unexpectedly found dream-produced topics: {pmeta!r}" + assert pmeta.get("topics") == [], f"proactive unexpectedly returned dream-produced topics: {pmeta!r}" print("\n" + "=" * 70) print("test_reme_e2e_full_loop passed") diff --git a/tests/unit/test_auto_dream.py b/tests/unit/test_auto_dream.py index a47f40f8..e87b82b8 100644 --- a/tests/unit/test_auto_dream.py +++ b/tests/unit/test_auto_dream.py @@ -18,9 +18,7 @@ from reme.schema import DreamState, FileNode from reme.steps.evolve.dream.extract import DreamExtractStep from reme.steps.evolve.dream.finish import DreamFinishStep from reme.steps.evolve.dream.integrate import DreamIntegrateStep, _snapshot_digest -from reme.steps.evolve.dream.proactive import ProactiveStep -from reme.steps.evolve.dream.topics import DreamTopicsStep -from reme.steps.evolve.dream.utils import load_yaml_topics, parse_structured_reply, recent_dates, scan_day_files +from reme.steps.evolve.dream.utils import parse_structured_reply, recent_dates, scan_day_files def _touch(path: Path, text: str = "x") -> Path: @@ -117,8 +115,8 @@ class _SequenceAgent(BaseAgentWrapper): return outcome -def test_scan_day_files_includes_nested_md_and_excludes_interests(): - """Scan day files.""" +def test_scan_day_files_includes_only_markdown_day_files(): + """Scan day indexes and nested Markdown notes without including YAML products.""" with tempfile.TemporaryDirectory() as tmp: workspace = Path(tmp) _touch(workspace / "daily" / "2026-05-28.md") @@ -171,6 +169,50 @@ def test_dream_extract_matches_posix_catalog_paths(tmp_path): asyncio.run(run()) +def test_dream_extract_removes_all_legacy_interests_entries_from_catalog(tmp_path): + """Auto Dream removes historical interests watermarks without touching exposure files.""" + + class Catalog(_Catalog): + """Catalog seeded with a legacy interests entry.""" + + def __init__(self, nodes): + super().__init__() + self.nodes = nodes + self.deleted = [] + + async def delete(self, path): + self.deleted.extend(path if isinstance(path, list) else [path]) + + async def get_nodes(self, paths=None): + return self.nodes + + async def run(): + interests = _touch(tmp_path / "daily" / "2026-05-28" / "interests.yaml", "topics: []\n") + old_interests = _touch(tmp_path / "daily" / "2026-01-01" / "interests.yaml", "topics: []\n") + rel_path = interests.relative_to(tmp_path).as_posix() + old_rel_path = old_interests.relative_to(tmp_path).as_posix() + catalog = Catalog( + [ + FileNode(path=rel_path, st_mtime=interests.stat().st_mtime), + FileNode(path=old_rel_path, st_mtime=old_interests.stat().st_mtime), + ], + ) + step = DreamExtractStep(scan_days=1, app_context=ApplicationContext(workspace_dir=str(tmp_path))) + + with patch("reme.steps.evolve.dream.extract.refresh_day_index", return_value={}): + response = await step( + RuntimeContext(date="2026-05-28", file_catalog=catalog, file_store=_FileStore(tmp_path)), + ) + + assert response.success is True + assert response.metadata["dream"]["deleted_paths"] == [old_rel_path, rel_path] + assert catalog.deleted == [old_rel_path, rel_path] + assert interests.read_text(encoding="utf-8") == "topics: []\n" + assert old_interests.read_text(encoding="utf-8") == "topics: []\n" + + asyncio.run(run()) + + def test_recent_dates_includes_anchor_and_previous_days(): """Recent date window is inclusive and chronological.""" assert recent_dates("2026-05-28", 3) == ["2026-05-26", "2026-05-27", "2026-05-28"] @@ -249,7 +291,7 @@ def test_extract_unusable_receipt_is_a_warning_not_a_failure(tmp_path): assert dream["units"] == [] assert dream["failed_paths"] == [] assert dream["warnings"] == [ - "dream extract skipped unusable agent receipt after retry; expected units and topics lists", + "dream extract skipped unusable agent receipt after retry; expected a units list", ] assert step.agent_wrapper.calls == 2 @@ -261,7 +303,7 @@ def test_extract_retries_one_unusable_receipt(tmp_path): async def run(): _touch(tmp_path / "daily" / "2026-05-28" / "session.md") - agent = _SequenceAgent({"result": "{}"}, {"result": '{"units": [], "topics": []}'}) + agent = _SequenceAgent({"result": "{}"}, {"result": '{"units": []}'}) step = DreamExtractStep( scan_days=1, app_context=ApplicationContext(workspace_dir=str(tmp_path)), @@ -474,278 +516,6 @@ def test_extract_without_llm_marks_changed_paths_failed(tmp_path): asyncio.run(run()) -def test_topics_step_writes_only_target_date_interests(): - """Topics are written only to ``state.date`` even when scan dates span multiple days.""" - - async def run(): - with tempfile.TemporaryDirectory() as tmp: - workspace = Path(tmp) - _touch(workspace / "daily" / "2026-05-26" / "old.md") - _touch(workspace / "daily" / "2026-05-28" / "today.md") - old_interests = workspace / "daily" / "2026-05-26" / "interests.yaml" - _touch(old_interests, "date: 2026-05-26\ntopics: []\n") - state = DreamState( - date="2026-05-28", - dates=["2026-05-26", "2026-05-27", "2026-05-28"], - workspace=str(workspace), - daily_dir="daily", - topics=[ - { - "title": "Old changed topic", - "reason": "Old daily material changed.", - "paths": ["daily/2026-05-26/old.md"], - }, - { - "title": "Today changed topic", - "reason": "Today's daily material changed.", - "paths": ["daily/2026-05-28/today.md"], - }, - ], - ) - step = DreamTopicsStep() - resp = await step(RuntimeContext(dream=state.model_dump(), file_store=_FileStore(workspace))) - - target = workspace / "daily" / "2026-05-28" / "interests.yaml" - dream = resp.metadata["dream"] - assert resp.success is True - assert target.is_file() - assert old_interests.read_text(encoding="utf-8") == "date: 2026-05-26\ntopics: []\n" - assert dream["interests_paths"] == ["daily/2026-05-28/interests.yaml"] - assert dream["modified_paths"] == ["daily/2026-05-28/interests.yaml"] - assert yaml.safe_load(target.read_text(encoding="utf-8"))["date"] == "2026-05-28" - - asyncio.run(run()) - - -def test_topics_same_content_is_not_modified(tmp_path): - """Rewriting deterministic interests content does not count as a user-visible change.""" - - async def run(): - topic = {"title": "Topic", "reason": "Reason", "paths": ["daily/source.md"]} - step = DreamTopicsStep() - - with patch("reme.steps.evolve.dream.topics.refresh_day_index", return_value={}): - first = await step( - RuntimeContext( - dream=DreamState( - date="2026-05-28", - workspace=str(tmp_path), - daily_dir="daily", - topics=[topic], - ).model_dump(), - file_store=_FileStore(tmp_path), - ), - ) - second = await step( - RuntimeContext( - dream=DreamState( - date="2026-05-28", - workspace=str(tmp_path), - daily_dir="daily", - topics=[topic], - ).model_dump(), - file_store=_FileStore(tmp_path), - ), - ) - third = await step( - RuntimeContext( - dream=DreamState( - date="2026-05-28", - workspace=str(tmp_path), - daily_dir="daily", - topics=[topic], - ).model_dump(), - file_store=_FileStore(tmp_path), - ), - ) - - assert first.metadata["dream"]["modified_paths"] == ["daily/2026-05-28/interests.yaml"] - assert second.metadata["dream"]["modified_paths"] == ["daily/2026-05-28/interests.yaml"] - assert third.metadata["dream"]["modified_paths"] == [] - - asyncio.run(run()) - - -def test_topics_agent_failure_falls_back_to_candidates(tmp_path): - """Topic ranking remains best-effort when the optional agent is unavailable.""" - - async def run(): - state = DreamState( - date="2026-05-28", - workspace=str(tmp_path), - daily_dir="daily", - topics=[{"title": "Topic", "reason": "Reason", "paths": ["daily/source.md"]}], - ) - step = DreamTopicsStep() - step.agent_wrapper = _ReplyAgent(error=RuntimeError("temporary model failure")) - - with ( - patch("reme.steps.evolve.dream.topics.refresh_day_index", return_value={}), - patch("reme.steps.evolve.dream.topics.llm_available", return_value=True), - ): - response = await step( - RuntimeContext( - dream=state.model_dump(), - file_store=_FileStore(tmp_path), - agent_wrapper=step.agent_wrapper, - ), - ) - - dream = response.metadata["dream"] - assert response.success is True - assert dream["topics_written"] == 1 - assert "deterministic fallback" in dream["warnings"][0] - - asyncio.run(run()) - - -def test_topics_does_not_overwrite_invalid_existing_yaml(tmp_path): - """A malformed user-owned interests file is preserved instead of treated as empty.""" - - async def run(): - target = _touch(tmp_path / "daily" / "2026-05-28" / "interests.yaml", "topics: [\n") - state = DreamState( - date="2026-05-28", - workspace=str(tmp_path), - daily_dir="daily", - topics=[{"title": "Topic", "reason": "Reason", "paths": ["daily/source.md"]}], - ) - step = DreamTopicsStep() - - response = await step(RuntimeContext(dream=state.model_dump(), file_store=_FileStore(tmp_path))) - - assert response.success is False - assert target.read_text(encoding="utf-8") == "topics: [\n" - assert "Invalid interests YAML" in response.answer - - asyncio.run(run()) - - -def test_topics_does_not_overwrite_invalid_existing_topic_entry(tmp_path): - """Strict loading rejects entries that lenient loading would discard.""" - - async def run(): - content = "topics:\n - title: User topic\n paths:\n - daily/source.md\n" - target = _touch(tmp_path / "daily" / "2026-05-28" / "interests.yaml", content) - state = DreamState( - date="2026-05-28", - workspace=str(tmp_path), - daily_dir="daily", - topics=[{"title": "New topic", "reason": "New reason", "paths": ["daily/source.md"]}], - ) - step = DreamTopicsStep() - - response = await step(RuntimeContext(dream=state.model_dump(), file_store=_FileStore(tmp_path))) - - assert response.success is False - assert target.read_text(encoding="utf-8") == content - assert "topics[0].reason must be a non-empty string" in response.answer - - asyncio.run(run()) - - -def test_strict_topic_loading_rejects_lossy_fields(tmp_path): - """Strict mode rejects values and fields that clean_topic would silently lose.""" - target = _touch(tmp_path / "interests.yaml", "topics:\n - title: Topic\n reason: Reason\n custom: keep me\n") - - try: - load_yaml_topics(target, strict=True) - except ValueError as exc: - assert "unknown field(s): custom" in str(exc) - else: - raise AssertionError("strict topic loading accepted a lossy field") - - -def test_strict_topic_loading_rejects_invalid_field_types(tmp_path): - """Strict mode rejects list fields that would otherwise be normalized away.""" - target = _touch( - tmp_path / "interests.yaml", - "topics:\n - title: Topic\n reason: Reason\n paths: daily/source.md\n", - ) - - try: - load_yaml_topics(target, strict=True) - except ValueError as exc: - assert "topics[0].paths must be a list of non-empty strings" in str(exc) - else: - raise AssertionError("strict topic loading accepted an invalid paths type") - - -def test_proactive_answer_includes_topics_and_requested_content(tmp_path): - """Successful proactive reads expose useful data through the primary answer.""" - - async def run(): - content = ( - "date: 2026-05-28\n" - "topics:\n" - " - title: Retrieval quality\n" - " reason: Search behavior changed repeatedly.\n" - " evidence: daily/2026-05-28/session.md\n" - ) - _touch(tmp_path / "daily" / "2026-05-28" / "interests.yaml", content) - step = ProactiveStep(app_context=ApplicationContext(workspace_dir=str(tmp_path))) - - response = await step(RuntimeContext(date="2026-05-28", include_content=True, file_store=_FileStore(tmp_path))) - - assert response.success is True - assert response.answer == { - "summary": "Read 1 proactive topic(s) from daily/2026-05-28/interests.yaml", - "topics": [ - { - "title": "Retrieval quality", - "reason": "Search behavior changed repeatedly.", - "evidence": "daily/2026-05-28/session.md", - "keywords": [], - "paths": [], - }, - ], - "content": content, - } - assert response.metadata["topics"] == response.answer["topics"] - assert response.metadata["content"] == content - - asyncio.run(run()) - - -def test_proactive_answer_omits_unrequested_content(tmp_path): - """Raw YAML is absent from the primary answer when include_content is false.""" - - async def run(): - _touch(tmp_path / "daily" / "2026-05-28" / "interests.yaml", "topics:\n - title: Topic\n reason: Reason\n") - step = ProactiveStep(app_context=ApplicationContext(workspace_dir=str(tmp_path))) - - response = await step(RuntimeContext(date="2026-05-28", include_content=False, file_store=_FileStore(tmp_path))) - - assert response.success is True - assert "content" not in response.answer - assert response.answer["topics"][0]["title"] == "Topic" - assert response.metadata["content"] == "" - - asyncio.run(run()) - - -def test_proactive_keeps_skipped_and_error_answers_explicit(tmp_path): - """Empty and failure outcomes remain distinguishable without reading metadata.""" - - async def run(): - step = ProactiveStep(app_context=ApplicationContext(workspace_dir=str(tmp_path))) - skipped = await step(RuntimeContext(date="2026-05-28", file_store=_FileStore(tmp_path))) - - assert skipped.success is True - assert skipped.answer == "Skipped: interests file not found at daily/2026-05-28/interests.yaml" - assert skipped.metadata["skipped"] is True - - _touch(tmp_path / "daily" / "2026-05-28" / "interests.yaml", "topics: []\n") - with patch("reme.steps.evolve.dream.proactive.load_yaml_topics", side_effect=ValueError("bad topics")): - failed = await step(RuntimeContext(date="2026-05-28", file_store=_FileStore(tmp_path))) - - assert failed.success is False - assert failed.answer == "Error: ValueError: bad topics" - assert failed.metadata["error"] == "ValueError: bad topics" - - asyncio.run(run()) - - def test_finish_does_not_checkpoint_failed_changed_paths(): """Finish does not checkpoint failed changed paths.""" @@ -755,7 +525,6 @@ def test_finish_does_not_checkpoint_failed_changed_paths(): ok = _touch(workspace / "daily" / "2026-05-28" / "ok.md") failed = _touch(workspace / "daily" / "2026-05-28" / "failed.md") day_index = _touch(workspace / "daily" / "2026-05-28.md") - interests = _touch(workspace / "daily" / "2026-05-28" / "interests.yaml") state = DreamState( date="2026-05-28", dates=["2026-05-26", "2026-05-27", "2026-05-28"], @@ -763,7 +532,6 @@ def test_finish_does_not_checkpoint_failed_changed_paths(): daily_dir="daily", changed_paths=[ok.relative_to(workspace).as_posix(), failed.relative_to(workspace).as_posix()], failed_paths=[failed.relative_to(workspace).as_posix()], - interests_paths=[interests.relative_to(workspace).as_posix()], modified_paths=["digest/procedure/example.md"], integrate_results=[ { @@ -785,7 +553,6 @@ def test_finish_does_not_checkpoint_failed_changed_paths(): assert "- [digest/procedure/example.md][CREATE]: Created a concise procedure node." in resp.answer assert ok.relative_to(workspace).as_posix() in upserted assert failed.relative_to(workspace).as_posix() not in upserted - assert interests.relative_to(workspace).as_posix() in upserted assert day_index.relative_to(workspace).as_posix() in upserted assert catalog.dumps == 1 assert resp.metadata["modified"] is True diff --git a/tests/unit/test_config_parser.py b/tests/unit/test_config_parser.py index 16ba54bf..726df2e7 100644 --- a/tests/unit/test_config_parser.py +++ b/tests/unit/test_config_parser.py @@ -109,6 +109,18 @@ def test_default_config_registers_workspace_web_jobs(): assert jobs["chat"]["steps"] == [{"backend": "chat_step", "agent_wrapper": "default"}] +def test_default_config_registers_manual_and_scheduled_proactive_refresh(): + """Proactive refresh is locally runnable while only the scheduled job starts automatically.""" + jobs = _load_config("default.yaml")["jobs"] + + manual = jobs["proactive_refresh"] + scheduled = jobs["proactive_refresh_cron"] + assert manual["backend"] == "base" + assert manual["enable_serve"] is False + assert scheduled["backend"] == "cron" + assert manual["steps"] == scheduled["steps"] + + def test_default_config_keeps_frontmatter_chunk_metadata_opt_in(): """Markdown frontmatter-to-chunk metadata is disabled by default for compatibility.""" cfg = _load_config("default.yaml") diff --git a/tests/unit/test_proactive_refresh.py b/tests/unit/test_proactive_refresh.py new file mode 100644 index 00000000..282ab55e --- /dev/null +++ b/tests/unit/test_proactive_refresh.py @@ -0,0 +1,1986 @@ +# pylint: disable=too-many-lines +"""Unit tests for the proactive refresh chain (PROACTIVE_SPEC.md A8 mapping). + +Coverage: F1 contracts, F2 discovery, F3 isolation, F4 budget/timeout, +F5 read extensions, plus the M2 extends branch and semantic dedup. +""" + +import asyncio +import datetime as dt +import os +import tempfile +import time +from pathlib import Path +from unittest.mock import patch + +import yaml +from agentscope.model import ChatModelBase + +from reme.components.agent_wrapper.base_agent_wrapper import BaseAgentWrapper +from reme.components.application_context import ApplicationContext +from reme.components.file_catalog import BaseFileCatalog +from reme.components.file_store import BaseFileStore +from reme.components.runtime_context import RuntimeContext +from reme.schema import FileNode, ProactiveTopic +from reme.steps.evolve.dream.extract import DreamExtractStep +from reme.steps.evolve.dream.utils import pack_paths +from reme.steps.evolve.proactive.agenda import ProactiveAgendaStep +from reme.steps.evolve.proactive.extract import ProactiveExtractStep +from reme.steps.evolve.proactive.finish import ProactiveFinishStep +from reme.steps.evolve.proactive.plan import ProactivePlanStep +from reme.steps.evolve.proactive.proactive import ProactiveStep +from reme.steps.evolve.proactive.topics import ProactiveTopicsStep +from reme.steps.evolve.proactive.utils import ( + load_carry_forward, + load_personal_profile_block, + load_state, + load_yaml_topics, + normalize_topic, + parse_interests_topics, + parse_extract_reply, + scan_material_daily, + strip_frontmatter, + topic_id, + trim_state_file, +) + +DAY = "2026-08-13" + + +def _touch(path: Path, text: str = "x") -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +class _Catalog(BaseFileCatalog): + """In-memory catalog stub recording every mutation.""" + + def __init__(self, nodes=None): + super().__init__() + self.nodes = list(nodes or []) + self.upserts = [] + self.dumps = 0 + self.deleted = [] + + async def upsert(self, nodes): + self.upserts.extend(nodes) + known = {n.path: n for n in self.nodes} + for node in nodes: + known[node.path] = node + self.nodes = list(known.values()) + + async def delete(self, path): + paths = path if isinstance(path, list) else [path] + self.deleted.extend(paths) + drop = set(paths) + self.nodes = [n for n in self.nodes if n.path not in drop] + + async def get_nodes(self, paths=None): + if paths is None: + return list(self.nodes) + wanted = set(paths) + return [n for n in self.nodes if n.path in wanted] + + async def dump(self): + self.dumps += 1 + + +class _FileStore(BaseFileStore): + def __init__(self, workspace: Path): + super().__init__() + self._workspace_path = workspace + + @property + def workspace_path(self) -> Path: + return self._workspace_path + + async def upsert(self, files): + return None + + async def delete(self, path): + return None + + async def clear(self): + return None + + async def get_nodes(self, paths=None): + return [] + + async def get_outlinks(self, path, scope=None): + return [] + + async def get_inlinks(self, path, scope=None): + return [] + + async def vector_search(self, query, limit, search_filter): + return [] + + async def keyword_search(self, query, limit, search_filter): + return [] + + +class _AgentWrapper(BaseAgentWrapper): + """Deterministic LLM stub with scripted replies and call accounting.""" + + def __init__(self, replies=None, delay: float = 0.0): + super().__init__() + self.replies = list(replies or []) + self.delay = delay + self.calls = 0 + self.last_inputs = [] + self.last_system_prompts = [] + + async def reply(self, inputs, **kwargs): + self.calls += 1 + self.last_inputs.append(inputs) + self.last_system_prompts.append(str(kwargs.get("system_prompt") or "")) + item = self.replies.pop(0) if self.replies else "" + if isinstance(item, tuple): + text, delay = item + else: + text, delay = item, self.delay + if delay: + await asyncio.sleep(delay) + return { + "session_id": "stub", + "result": text, + "last_message": {"content": [{"type": "text", "text": text}]}, + } + + +class _DummyModel(ChatModelBase): + """Placeholder model satisfying dream extract's llm_available check.""" + + def __init__(self): + pass + + +class _FakeEmbedding: + """Scripted embedding oracle: exact text -> vector, else orthogonal default. + + Declares ``kwargs``/``dimensions`` so the topics step can fingerprint the + "model" against ``known_threshold_calibrated_for`` (defaults match). + """ + + def __init__(self, vectors: dict[str, list[float]], model: str = "text-embedding-v4", dimensions: int = 1024): + self.vectors = vectors + self.calls = 0 + self.kwargs = {"model": model} + self.dimensions = dimensions + + async def __call__(self, inputs, **kwargs): + self.calls += 1 + return [list(self.vectors.get(text, [0.0, 1.0])) for text in inputs] + + +def _reply(follow_ups=None, extends=None, updates=None) -> str: + doc = {} + if follow_ups is not None: + doc["follow_ups"] = follow_ups + if extends is not None: + doc["extends"] = extends + if updates is not None: + doc["updates"] = updates + body = yaml.safe_dump(doc, allow_unicode=True, sort_keys=False) + return f"```yaml\n{body}```" + + +def _topic(title, reason="because", confidence=0.7, paths=None, evidence=None): + doc = { + "title": title, + "reason": reason, + "confidence": confidence, + "paths": paths or [], + } + if evidence: + doc["evidence"] = evidence + return doc + + +def _state_topic(title, day=DAY, kind="follow_up", confidence=0.8, topic_id_value=None, status="open"): + return { + "id": topic_id_value or topic_id(title), + "title": title, + "reason": "seeded", + "kind": kind, + "confidence": confidence, + "status": status, + "first_seen": day, + "last_evidence_at": day, + "evidence": "", + "paths": [], + } + + +def _write_state(ws, budget=None, exposure=None, open_topics=None, resolved=None, omit_open_topics=False): + data = {"version": 1} + if budget is not None: + data["budget"] = budget + if exposure is not None: + data["exposure"] = exposure + if not omit_open_topics: + data["open_topics"] = open_topics or [] + if resolved is not None: + data["resolved"] = resolved + _touch(ws / "daily" / "_proactive.yaml", yaml.safe_dump(data, allow_unicode=True, sort_keys=False)) + + +def _read_state(ws) -> dict: + return yaml.safe_load((ws / "daily" / "_proactive.yaml").read_text(encoding="utf-8")) + + +def _interests(ws, day=DAY) -> Path: + return ws / "daily" / day / "interests.yaml" + + +def _write_interests_v1(ws, day, topics_yaml: list[dict]): + payload = {"date": day, "topic_count": len(topics_yaml), "topics": topics_yaml} + _touch(_interests(ws, day), yaml.safe_dump(payload, allow_unicode=True, sort_keys=False)) + + +def _write_interests_v2(ws, day, topics: list[dict], push=True, skip_reason=""): + payload = { + "version": 2, + "date": day, + "generated_at": f"{day}T09:00:00+08:00", + "push": push, + "skip_reason": skip_reason, + "topics": topics, + } + _touch(_interests(ws, day), yaml.safe_dump(payload, allow_unicode=True, sort_keys=False)) + + +async def _run_chain( + ws, + replies=None, + *, + wrapper=None, + catalog=None, + app=None, + extract_kwargs=None, + topics_kwargs=None, + extra_ctx=None, + context=None, +): + app = app or ApplicationContext(workspace_dir=str(ws)) + wrapper = wrapper or _AgentWrapper(replies or []) + catalog = catalog or _Catalog() + context = context or RuntimeContext( + date=DAY, + file_catalog=catalog, + file_store=_FileStore(ws), + agent_wrapper=wrapper, + **(extra_ctx or {}), + ) + extract = ProactiveExtractStep(app_context=app, **(extract_kwargs or {})) + resp_extract = await extract(context) + topics = ProactiveTopicsStep(app_context=app, **(topics_kwargs or {})) + resp_topics = await topics(context) + finish = ProactiveFinishStep(app_context=app) + resp_finish = await finish(context) + return app, wrapper, catalog, context, (resp_extract, resp_topics, resp_finish) + + +def test_known_threshold_model_binding(tmp_path): + """known_threshold is model-bound (v5.1): fingerprint mismatch degrades. + + The oracle maps every text to the same vector (sim 1.0 >= known_threshold), + but the "model" differs from the calibration, so the semantic gate must be + disabled: a paraphrase survives while an exact normalize duplicate of a + historical interests title still drops. + """ + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "notes") + _write_state(ws, open_topics=[_state_topic("检索引用质量评测")]) + _write_interests_v1(ws, "2026-08-12", [{"title": "向量数据库选型", "reason": "done"}]) + embedder = _FakeEmbedding({}, model="some-other-model-v9") + reply = _reply( + follow_ups=[ + _topic("检索引用可信度的评测方法", paths=[f"daily/{DAY}/session.md"]), + _topic("向量数据库选型", paths=[f"daily/{DAY}/session.md"]), + ], + ) + _, _, _, context, _ = await _run_chain(ws, [reply], extra_ctx={"as_embedding": embedder}) + state = context.get("proactive") + assert state["dropped_known"] == 0 # semantic gate disabled by mismatch + assert state["dropped_duplicate"] == 1 # exact compare still active + titles = [t["title"] for t in _read_state(ws)["open_topics"]] + assert "检索引用可信度的评测方法" in titles + assert "向量数据库选型" not in titles + + asyncio.run(run()) + + +def test_overage_remention_restarts(tmp_path): + """Over-age open topics re-mentioned today restart instead of vanishing. + + Same-id candidate refreshes first_seen so trim keeps the topic (unfinished + business must keep being re-executed). After the restart the revived title + is back in the comparison set, so a paraphrase of it drops as known. + A dying topic with NO same-id candidate stays excluded from comparison: + its paraphrase re-mention survives and opens a fresh topic. + """ + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "notes") + old_day = "2026-07-28" # 16 days before DAY, over-age at carry_forward_days=14 + _write_state( + ws, + open_topics=[_state_topic("旧任务确认", day=old_day), _state_topic("遗留事项", day=old_day)], + ) + # Discriminative oracle: only the legacy-task / its-supplement pair scores sim 1.0; + # every other pair is orthogonal. If the dying title leaked into the + # comparison set, the paraphrase below would drop as known. + embedder = _FakeEmbedding({"遗留事项的补充。because": [1.0, 0.0], "遗留事项。seeded": [1.0, 0.0]}) + reply = _reply( + follow_ups=[ + _topic("旧任务确认", paths=[f"daily/{DAY}/session.md"]), + _topic("旧任务确认的后续安排", paths=[f"daily/{DAY}/session.md"]), + _topic("遗留事项的补充", paths=[f"daily/{DAY}/session.md"]), + ], + ) + _, _, _, context, _ = await _run_chain(ws, [reply], extra_ctx={"as_embedding": embedder}) + state = context.get("proactive") + assert state["dropped_known"] == 1 # paraphrase of the REVIVED topic (default sim 1.0 vs it) + topics = {t["id"]: t for t in _read_state(ws)["open_topics"]} + assert topics[topic_id("旧任务确认")]["first_seen"] == DAY # restarted, not trimmed + assert topic_id("旧任务确认的后续安排") not in topics # known vs revived topic + assert topic_id("遗留事项的补充") in topics # dying title excluded from comparison + assert topic_id("遗留事项") not in topics # over-age, never re-mentioned -> trimmed + + asyncio.run(run()) + + +def test_update_evidence_anchor(tmp_path): + """v5.2: action=update must anchor this round's new material (hard check). + + An update citing a file outside the round's changed material degrades to + keep (no evidence rewrite, no freshness refresh); an update citing a + changed file applies, and last_evidence_at is parsed from the evidence + path date instead of defaulting to today. + """ + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "today notes") + _touch(ws / "daily" / "2026-08-12" / "old-note.md", "yesterday notes") + _write_state( + ws, + open_topics=[_state_topic("证据改挂主题"), _state_topic("昨日证据主题")], + ) + reply = _reply( + updates=[ + {"id": topic_id("证据改挂主题"), "action": "update", "evidence": "daily/2026-08-11/absent.md"}, + { + "id": topic_id("昨日证据主题"), + "action": "update", + "evidence": "daily/2026-08-12/old-note.md#L5", + "confidence": 0.9, + }, + ], + ) + _, _, _, context, _ = await _run_chain(ws, [reply]) + state = context.get("proactive") + assert state["updates_applied"] == 1 + topics = {t["id"]: t for t in _read_state(ws)["open_topics"]} + rejected = topics[topic_id("证据改挂主题")] + assert rejected["last_evidence_at"] == DAY # untouched, degraded to keep + assert rejected["evidence"] != "daily/2026-08-11/absent.md" + applied = topics[topic_id("昨日证据主题")] + assert applied["last_evidence_at"] == "2026-08-12" # parsed from evidence path + assert applied["evidence"] == "daily/2026-08-12/old-note.md#L5" + assert applied["confidence"] == 0.9 + + asyncio.run(run()) + + +def test_no_embedding_configured(tmp_path): + """No as_embedding anywhere: the chain runs and dedups exactly (BM25-only). + + Paraphrase-level candidates survive (no semantic gate), exact normalize + duplicates of historical exposure still drop; nothing crashes or skips. + """ + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "notes") + _write_state(ws, open_topics=[_state_topic("已知主题")]) + _write_interests_v1(ws, "2026-08-12", [{"title": "历史曝光主题", "reason": "x"}]) + reply = _reply( + follow_ups=[ + _topic("已知主题的不同说法", paths=[f"daily/{DAY}/session.md"]), + _topic("历史曝光主题", paths=[f"daily/{DAY}/session.md"]), + ], + ) + _, _, _, context, _ = await _run_chain(ws, [reply]) # no as_embedding in context/app + state = context.get("proactive") + assert state["dropped_known"] == 0 + assert state["dropped_duplicate"] == 1 + titles = [t["title"] for t in _read_state(ws)["open_topics"]] + assert "已知主题的不同说法" in titles + assert "历史曝光主题" not in titles + + asyncio.run(run()) + + +# --------------------------------------------------------------------------- +# F1 contracts +# --------------------------------------------------------------------------- + + +def test_normalize_and_topic_id_frozen(): + """A7 frozen contract: NFKC+casefold, keep L/N only; id is a 12-hex sha1 prefix.""" + assert normalize_topic("Memory Search: 可解释性评估!") == "memorysearch可解释性评估" + assert topic_id("A") == topic_id("a") == topic_id(" A ") + assert topic_id("A") != topic_id("B") + assert len(topic_id("anything")) == 12 + + +def test_legacy_file_parse(tmp_path): + """v1 files parse without error; new fields take deterministic defaults (A2).""" + content = ( + 'date: "2026-08-13"\n' + "topic_count: 1\n" + "topics:\n" + " - title: Retrieval quality\n" + " reason: Search behavior changed.\n" + " evidence: daily/2026-08-13/session.md\n" + ) + _touch(_interests(tmp_path), content) + topics, is_v1, push = parse_interests_topics(yaml.safe_load(content), DAY) + assert is_v1 is True and push is True + topic = topics[0] + assert topic.id == topic_id("Retrieval quality") + assert topic.kind == "interest_extend" + assert topic.confidence == 0.5 + assert topic.first_seen == DAY + assert topic.last_evidence_at == DAY + + async def run(): + step = ProactiveStep(app_context=ApplicationContext(workspace_dir=str(tmp_path))) + response = await step(RuntimeContext(date=DAY, include_content=False, file_store=_FileStore(tmp_path))) + assert response.success is True + assert response.metadata["push"] is True + + asyncio.run(run()) + + +def test_field_fallback(): + """Illegal kind/confidence values fall back deterministically.""" + topic = ProactiveTopic(title="T", reason="R", kind="bogus", confidence="not-a-number") + assert topic.kind == "interest_extend" + assert topic.confidence == 0.5 + assert ProactiveTopic(confidence=1.7).confidence == 1.0 + assert ProactiveTopic(confidence=-3).confidence == 0.0 + + +def test_resolved_registry(tmp_path): + """Resolved ids disappear from carry-forward and from F5 reads.""" + + async def run(): + ws = tmp_path + id_a, id_b = topic_id("Topic A"), topic_id("Topic B") + _write_state( + ws, + open_topics=[_state_topic("Topic A", topic_id_value=id_a), _state_topic("Topic B", topic_id_value=id_b)], + resolved=[{"id": id_b, "title": "Topic B", "resolved_at": DAY, "evidence": ""}], + ) + state_file, needs_bootstrap = load_state(ws) + assert needs_bootstrap is False + carry_all, _ = await load_carry_forward(ws, state_file, DAY, 14, 20) + assert [t.title for t in carry_all] == ["Topic A"] + + _write_interests_v2( + ws, + DAY, + [_state_topic("Topic A", topic_id_value=id_a), _state_topic("Topic B", topic_id_value=id_b)], + ) + step = ProactiveStep(app_context=ApplicationContext(workspace_dir=str(ws))) + response = await step(RuntimeContext(date=DAY, include_content=False, file_store=_FileStore(ws))) + assert [t["title"] for t in response.answer["topics"]] == ["Topic A"] + + asyncio.run(run()) + + +def test_carry_forward_bootstrap(tmp_path): + """First run (missing state file) bootstraps from interests.yaml history; first_seen = min.""" + + async def run(): + ws = tmp_path + _write_interests_v1(ws, "2026-08-11", [_topic("Topic T", paths=["daily/2026-08-11/a.md"])]) + _write_interests_v1( + ws, + "2026-08-12", + [_topic("Topic T", paths=["daily/2026-08-12/a.md"]), _topic("Topic U", paths=["daily/2026-08-12/b.md"])], + ) + + # Upgrade path: no _proactive.yaml yet -> one-time bootstrap from history. + state_file, needs_bootstrap = load_state(ws) + assert needs_bootstrap is True + carry_all, _ = await load_carry_forward(ws, state_file, DAY, 14, 20, needs_bootstrap=needs_bootstrap) + by_title = {t.title: t for t in carry_all} + assert set(by_title) == {"Topic T", "Topic U"} + assert by_title["Topic T"].first_seen == "2026-08-11" # min across days, not latest + assert by_title["Topic U"].first_seen == "2026-08-12" + + persisted = _read_state(ws) + assert len(persisted["open_topics"]) == 2 + _, needs_bootstrap2 = load_state(ws) + assert needs_bootstrap2 is False # no re-bootstrap once the key exists + + # Empty open_topics list is a normal state, not a bootstrap trigger. + _write_state(ws, open_topics=[]) + _, needs_bootstrap3 = load_state(ws) + assert needs_bootstrap3 is False + + # An existing file lacking the key (e.g. hand-edited) still triggers it. + _write_state(ws, budget={}, omit_open_topics=True) + _, needs_bootstrap4 = load_state(ws) + assert needs_bootstrap4 is True + + asyncio.run(run()) + + +def test_state_file_corrupt_no_bootstrap(tmp_path): + """Corrupt _proactive.yaml rebuilds empty WITHOUT bootstrap (spec F1.3/A2/A5).""" + + async def run(): + ws = tmp_path + _write_interests_v1(ws, "2026-08-12", [_topic("Topic U", paths=["daily/2026-08-12/b.md"])]) + _touch(ws / "daily" / "_proactive.yaml", "version: [unclosed") + + state_file, needs_bootstrap = load_state(ws) + assert needs_bootstrap is False + assert state_file.open_topics == [] + carry_all, _ = await load_carry_forward(ws, state_file, DAY, 14, 20, needs_bootstrap=needs_bootstrap) + assert carry_all == [] + + asyncio.run(run()) + + +def test_bootstrap_upgrade_chain(tmp_path): + """Upgrade path: the first refresh run seeds the truth source from interests history.""" + + async def run(): + ws = tmp_path + _write_interests_v1(ws, "2026-08-12", [_topic("历史主题", paths=["daily/2026-08-12/a.md"])]) + _touch(ws / "daily" / DAY / "s1.md", "new evidence") + reply = _reply(follow_ups=[_topic("新主题", confidence=0.9, paths=[f"daily/{DAY}/s1.md"])]) + _, _, _, context, responses = await _run_chain(ws, [reply]) + assert all(r.success for r in responses) + + assert context.get("proactive")["carry_forward_count"] == 1 # history reached the chain context + + truth_titles = {t["title"] for t in _read_state(ws)["open_topics"]} + assert truth_titles == {"历史主题", "新主题"} + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert {t["title"] for t in data["topics"]} == {"历史主题", "新主题"} + + asyncio.run(run()) + + +def test_carry_forward_expiry(tmp_path): + """Topics older than carry_forward_days drop from carry-forward and are pruned.""" + + async def run(): + ws = tmp_path + _write_state(ws, open_topics=[_state_topic("Old Topic", day="2026-07-01")]) + state_file, _ = load_state(ws) + carry_all, _ = await load_carry_forward(ws, state_file, DAY, 14, 20) + assert carry_all == [] + + _touch(ws / "daily" / DAY / "note.md", "new evidence") + reply = _reply(follow_ups=[], updates=[]) + _, _, _, context, responses = await _run_chain(ws, [reply]) + assert all(r.success for r in responses) + assert context.get("proactive")["carry_forward_count"] == 0 + assert _read_state(ws)["open_topics"] == [] # pruned from the truth source + + asyncio.run(run()) + + +def test_material_filter(tmp_path): + """Day indexes, ``_*`` files and interests.yaml never enter the material set (INV-11).""" + ws = tmp_path + _touch(ws / "daily" / f"{DAY}.md", "day index") + _touch(ws / "daily" / DAY / "_hidden.md", "underscore file") + _touch(ws / "daily" / DAY / "interests.yaml", "topics: []\n") + _touch(ws / "daily" / DAY / "good.md", "material") + assert scan_material_daily(ws, DAY, "daily", 2) == [f"daily/{DAY}/good.md"] + + +# --------------------------------------------------------------------------- +# F2 discovery + chain behaviour +# --------------------------------------------------------------------------- + + +def test_refresh_chain_e2e(tmp_path): + """extract -> topics -> finish runs end to end.""" + + async def run(): + ws = tmp_path + note = _touch(ws / "daily" / DAY / "session.md", "we discussed the eval plan; nothing landed") + reply = _reply( + follow_ups=[_topic("记忆检索的可解释性评估", confidence=0.9, paths=["daily/2026-08-13/session.md"])], + ) + app = ApplicationContext(workspace_dir=str(ws)) + wrapper = _AgentWrapper([reply]) + catalog = _Catalog() + context = RuntimeContext(date=DAY, file_catalog=catalog, file_store=_FileStore(ws), agent_wrapper=wrapper) + + extract = ProactiveExtractStep(app_context=app) + resp_extract = await extract(context) + topics = ProactiveTopicsStep(app_context=app) + resp_topics = await topics(context) + finish = ProactiveFinishStep(app_context=app) + resp_finish = await finish(context) + + assert resp_extract.success and resp_topics.success and resp_finish.success + state = context.get("proactive") + assert state["llm_calls"] == 1 + assert state["push"] is True + assert state["file_skip_reason"] == "" + assert state["interests_written"] is True + + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert data["version"] == 2 + assert data["push"] is True + assert data["topics"][0]["title"] == "记忆检索的可解释性评估" + assert data["topics"][0]["kind"] == "follow_up" + + truth = _read_state(ws) + assert len(truth["open_topics"]) == 1 + checkpoint_paths = {n.path for n in catalog.upserts} + assert checkpoint_paths == {note.relative_to(ws).as_posix()} # interests.yaml not checkpointed (R6) + assert catalog.dumps == 1 + + asyncio.run(run()) + + +def test_open_loop_follow_up(tmp_path): + """Branch A emits follow_ups with stable ids; paths restricted to M.""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "open loop content") + reply = _reply( + follow_ups=[ + _topic("未决事项甲", confidence=0.9, paths=[f"daily/{DAY}/session.md"]), + _topic("越界事项", confidence=0.9, paths=["daily/2026-08-01/absent.md"]), + ], + ) + _, _, _, context, _ = await _run_chain(ws, [reply]) + state = context.get("proactive") + assert len(state["follow_ups"]) == 1 + follow_up = state["follow_ups"][0] + assert follow_up["kind"] == "follow_up" + assert follow_up["id"] == topic_id("未决事项甲") + assert state["dropped_missing"] == 1 # out-of-M paths dropped, never repaired + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert [t["title"] for t in data["topics"]] == ["未决事项甲"] + + asyncio.run(run()) + + +def test_updates_actions(tmp_path): + """keep/update/resolve act on the truth source; unknown ids ignored.""" + + async def run(): + ws = tmp_path + id_a, id_b, id_c = topic_id("Topic A"), topic_id("Topic B"), topic_id("Topic C") + _write_state( + ws, + open_topics=[ + {**_state_topic("Topic A", day="2026-08-12", topic_id_value=id_a)}, + {**_state_topic("Topic B", day="2026-08-12", topic_id_value=id_b)}, + {**_state_topic("Topic C", day="2026-08-12", topic_id_value=id_c)}, + ], + ) + _touch(ws / "daily" / DAY / "session.md", "new evidence today") + evidence = f"daily/{DAY}/session.md" + reply = _reply( + follow_ups=[], + updates=[ + {"id": id_a, "action": "keep"}, + {"id": id_b, "action": "update", "evidence": evidence}, + {"id": id_c, "action": "resolve", "evidence": evidence}, + {"id": "ffffffffffff", "action": "resolve"}, + ], + ) + _, _, _, context, responses = await _run_chain(ws, [reply]) + assert all(r.success for r in responses) + state = context.get("proactive") + assert state["updates_applied"] == 1 + assert state["updates_resolved"] == 1 + + truth = _read_state(ws) + by_id = {t["id"]: t for t in truth["open_topics"]} + assert set(by_id) == {id_a, id_b} + assert by_id[id_a]["last_evidence_at"] == "2026-08-12" # keep: untouched + assert by_id[id_b]["last_evidence_at"] == DAY + assert by_id[id_b]["evidence"] == evidence + resolved_ids = {r["id"] for r in truth["resolved"]} + assert resolved_ids == {id_c} + + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert {t["title"] for t in data["topics"]} == {"Topic A", "Topic B"} + + asyncio.run(run()) + + +def test_resolved_resurrect(tmp_path): + """A candidate hitting a tombstone resurrects it: tombstone removed, first_seen kept.""" + + async def run(): + ws = tmp_path + tid = topic_id("Revived Topic") + _write_state( + ws, + resolved=[ + { + "id": tid, + "title": "Revived Topic", + "resolved_at": "2026-08-12", + "first_seen": "2026-08-08", + "evidence": "", + }, + ], + ) + _touch(ws / "daily" / DAY / "session.md", "the plan was reopened today") + reply = _reply(follow_ups=[_topic("Revived Topic", confidence=0.7, paths=[f"daily/{DAY}/session.md"])]) + _, _, _, context, _ = await _run_chain(ws, [reply]) + state = context.get("proactive") + assert [c["title"] for c in state["candidates"]] == ["Revived Topic"] + + truth = _read_state(ws) + assert truth["resolved"] == [] # tombstone removed + by_id = {t["id"]: t for t in truth["open_topics"]} + assert tid in by_id + assert by_id[tid]["first_seen"] == "2026-08-08" # original age anchor kept + assert by_id[tid]["last_evidence_at"] == DAY + + asyncio.run(run()) + + +def test_trim_freshness_first(tmp_path): + """max_topics trims by the freshness-first order, regardless of kind (v5 aging).""" + + async def run(): + ws = tmp_path + stale = [ + {**_state_topic(f"Stale Follow {i}", day="2026-08-05", kind="follow_up"), "last_evidence_at": "2026-08-05"} + for i in range(3) + ] + fresh = [_state_topic(f"Fresh Extend {i}", day=DAY, kind="interest_extend") for i in range(2)] + _write_state(ws, open_topics=stale + fresh) + _touch(ws / "daily" / DAY / "session.md", "material") + reply = _reply(follow_ups=[_topic("New Follow", confidence=0.9, paths=[f"daily/{DAY}/session.md"])]) + _, _, _, context, _ = await _run_chain(ws, [reply], topics_kwargs={"max_topics": 4}) + titles = [t["title"] for t in context.get("proactive")["topics_out"]] + assert len(titles) == 4 + assert titles[0] == "New Follow" # today's follow_up first + assert set(titles[1:3]) == {"Fresh Extend 0", "Fresh Extend 1"} # today's extends next + assert titles[3].startswith("Stale Follow") # only one stale topic survives the cap + + asyncio.run(run()) + + +def test_dedup_fallback(tmp_path): + """M1 fallback: normalize_topic exact match against recent interests drops duplicates.""" + + async def run(): + ws = tmp_path + _write_interests_v1(ws, "2026-08-12", [_topic("Retrieval Evaluation", paths=["daily/2026-08-12/a.md"])]) + # Seed an empty truth source so the history topic stays out of open_topics + # (otherwise first-run bootstrap would merge the same-id candidate instead + # of dropping it as a duplicate). + _write_state(ws, open_topics=[]) + _touch(ws / "daily" / DAY / "session.md", "material") + reply = _reply( + follow_ups=[_topic("retrieval evaluation!!", confidence=0.9, paths=[f"daily/{DAY}/session.md"])], + ) + _, _, _, context, _ = await _run_chain(ws, [reply]) + state = context.get("proactive") + assert state["dropped_duplicate"] == 1 + assert state["candidates"] == [] + assert state["file_skip_reason"] == "all_duplicates" # metadata-only since v5 (R7) + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert data["push"] is False + assert data["topics"] == [] + + asyncio.run(run()) + + +def test_skip_low_confidence(tmp_path): + """Candidates under min_push_confidence persist but are not pushed.""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "weak signal") + reply = _reply(follow_ups=[_topic("弱信号主题", confidence=0.3, paths=[f"daily/{DAY}/session.md"])]) + _, _, _, context, _ = await _run_chain(ws, [reply]) + state = context.get("proactive") + assert state["push"] is False + assert state["file_skip_reason"] == "low_confidence" + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert data["push"] is False + assert "skip_reason" not in data # metadata-only since v5 (R7) + assert [t["title"] for t in data["topics"]] == ["弱信号主题"] # rendered from truth source + assert [t["title"] for t in _read_state(ws)["open_topics"]] == ["弱信号主题"] # persisted + + asyncio.run(run()) + + +def test_no_new_evidence_zero_llm(tmp_path): + """No changed material -> early exit with zero LLM calls.""" + + async def run(): + ws = tmp_path + note = _touch(ws / "daily" / DAY / "session.md", "already seen") + rel = note.relative_to(ws).as_posix() + catalog = _Catalog([FileNode(path=rel, st_mtime=note.stat().st_mtime)]) + wrapper = _AgentWrapper(["should never be consumed"]) + _, wrapper, catalog, context, responses = await _run_chain( + ws, + wrapper=wrapper, + catalog=catalog, + ) + state = context.get("proactive") + assert state["early_exit"] == "no_new_evidence" + assert wrapper.calls == 0 + assert not _interests(ws).exists() + assert catalog.upserts == [] and catalog.dumps == 0 + assert all(r.success for r in responses) + + asyncio.run(run()) + + +def test_push_sticky(tmp_path): + """Within one day push only goes up: derived from the cumulative truth source (R1).""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "s1.md", "strong evidence") + reply1 = _reply(follow_ups=[_topic("高置信主题", confidence=0.9, paths=[f"daily/{DAY}/s1.md"])]) + _, _, _, context1, _ = await _run_chain(ws, [reply1]) + assert context1.get("proactive")["push"] is True + assert yaml.safe_load(_interests(ws).read_text(encoding="utf-8"))["push"] is True + + _touch(ws / "daily" / DAY / "s2.md", "weak evidence") + reply2 = _reply(follow_ups=[_topic("低置信主题", confidence=0.3, paths=[f"daily/{DAY}/s2.md"])]) + _, _, _, context2, _ = await _run_chain(ws, [reply2]) + state2 = context2.get("proactive") + assert state2["push"] is True # sticky: the 0.9 topic discovered today persists in truth source + assert state2["file_skip_reason"] == "" + assert yaml.safe_load(_interests(ws).read_text(encoding="utf-8"))["push"] is True + + asyncio.run(run()) + + +def test_idempotent_write(tmp_path): + """Same render skips writing; nightly v1 is adopted via bootstrap (v5 R1).""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "s1.md", "evidence one") + reply1 = _reply(follow_ups=[_topic("主题甲", confidence=0.9, paths=[f"daily/{DAY}/s1.md"])]) + _, _, _, context1, _ = await _run_chain(ws, [reply1]) + assert context1.get("proactive")["interests_written"] is True + mtime_before = _interests(ws).stat().st_mtime + + _touch(ws / "daily" / DAY / "s2.md", "evidence two, no new topics") + reply2 = _reply(follow_ups=[], updates=[]) + time.sleep(0.01) + _, _, _, context2, _ = await _run_chain(ws, [reply2]) + assert context2.get("proactive")["interests_written"] is False # identical render + assert _interests(ws).stat().st_mtime == mtime_before + + asyncio.run(run()) + + async def run_v1_nightly_adoption(): + """v5 R1: no v1 guard - first-run bootstrap adopts the nightly topic.""" + with tempfile.TemporaryDirectory() as tmp: + ws = Path(tmp) + nightly = [ + { + "title": "Nightly topic", + "reason": "written by dream", + "evidence": f"daily/{DAY}/n.md", + "paths": [f"daily/{DAY}/n.md"], + }, + ] + _write_interests_v1(ws, DAY, nightly) + _touch(ws / "daily" / DAY / "s1.md", "material") + reply = _reply(follow_ups=[], updates=[]) + _, _, _, context, _ = await _run_chain(ws, [reply]) + state = context.get("proactive") + # Bootstrap gives the nightly topic first_seen = file date (today); + # push is derived true because its fallback confidence 0.5 passes. + assert state["push"] is True + assert state["interests_written"] is True + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert data["version"] == 2 + assert [t["title"] for t in data["topics"]] == ["Nightly topic"] + + asyncio.run(run_v1_nightly_adoption()) + + +# --------------------------------------------------------------------------- +# F3 isolation / interleave +# --------------------------------------------------------------------------- + + +def test_refresh_isolation(tmp_path): + """Refresh never writes digest, never touches the dream catalog or day index.""" + + async def run(): + ws = tmp_path + digest_file = _touch(ws / "digest" / "wiki" / "existing.md", "digest content") + day_index = _touch(ws / "daily" / f"{DAY}.md", "index content") + _touch(ws / "daily" / DAY / "session.md", "conversation") + reply = _reply(follow_ups=[_topic("隔离主题", confidence=0.9, paths=[f"daily/{DAY}/session.md"])]) + proactive_catalog = _Catalog() + dream_catalog = _Catalog() + _, _, _, _, responses = await _run_chain(ws, [reply], catalog=proactive_catalog) + assert all(r.success for r in responses) + + assert digest_file.read_text(encoding="utf-8") == "digest content" + assert day_index.read_text(encoding="utf-8") == "index content" + assert not dream_catalog.upserts and dream_catalog.dumps == 0 and not dream_catalog.deleted + assert proactive_catalog.dumps == 1 + assert {n.path for n in proactive_catalog.upserts} == {f"daily/{DAY}/session.md"} + + asyncio.run(run()) + + +def test_nightly_overwrite_continuity(tmp_path): + """Nightly v1 overwrite cannot break continuity: truth source restores exposure.""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "s1.md", "round one") + reply1 = _reply(follow_ups=[_topic("持续主题", confidence=0.9, paths=[f"daily/{DAY}/s1.md"])]) + await _run_chain(ws, [reply1]) + truth_before = _read_state(ws)["open_topics"] + + # Nightly dream overwrites the exposure product with a v1 file. + _write_interests_v1(ws, DAY, [_topic("Nightly only", paths=[f"daily/{DAY}/s1.md"])]) + + _touch(ws / "daily" / DAY / "s2.md", "round two") + reply2 = _reply(follow_ups=[], updates=[]) + _, _, _, context, _ = await _run_chain(ws, [reply2]) + assert context.get("proactive")["interests_written"] is True + + assert _read_state(ws)["open_topics"] == truth_before # truth source intact + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert data["version"] == 2 # exposure restored from the truth source + assert [t["title"] for t in data["topics"]] == ["持续主题"] + + asyncio.run(run()) + + +def test_refresh_nightly_interleave(tmp_path): + """Refresh and nightly writes interleave without corrupting state or continuity.""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "s1.md", "round one") + reply1 = _reply(follow_ups=[_topic("连续主题", confidence=0.9, paths=[f"daily/{DAY}/s1.md"])]) + await _run_chain(ws, [reply1]) + + # Nightly dream overwrites interests.yaml with a v1 file. + _write_interests_v1(ws, DAY, [_topic("Nightly topic", paths=[f"daily/{DAY}/s1.md"])]) + nightly = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert nightly.get("version") is None + + # Next refresh round re-renders from the intact truth source. + _touch(ws / "daily" / DAY / "s2.md", "round two") + reply2 = _reply(follow_ups=[], updates=[]) + _, _, _, _, responses = await _run_chain(ws, [reply2]) + assert all(r.success for r in responses) + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert data["version"] == 2 + assert data["push"] is True + assert [t["title"] for t in data["topics"]] == ["连续主题"] + assert [t["title"] for t in _read_state(ws)["open_topics"]] == ["连续主题"] + + asyncio.run(run()) + + +def test_nightly_unit_set_invariant(): + """Dream extract sees identical units whether or not refresh ran first (INV-9).""" + + dream_reply = ( + "```yaml\n" + "units:\n" + " - name: eval-plan-gap\n" + " bucket: wiki\n" + " summary: The evaluation plan is discussed but never lands.\n" + f" paths: [daily/{DAY}/session.md]\n" + "topics: []\n" + "```" + ) + + async def dream_units(ws) -> list[dict]: + catalog = _Catalog() + wrapper = _AgentWrapper([dream_reply]) + step = DreamExtractStep(scan_days=1, app_context=ApplicationContext(workspace_dir=str(ws))) + with patch("reme.steps.evolve.dream.extract.refresh_day_index", return_value={}): + response = await step( + RuntimeContext( + date=DAY, + file_catalog=catalog, + file_store=_FileStore(ws), + agent_wrapper=wrapper, + as_llm=_DummyModel(), + ), + ) + assert response.success is True + return response.metadata["dream"]["units"] + + async def run(): + with tempfile.TemporaryDirectory() as tmp_a, tempfile.TemporaryDirectory() as tmp_b: + ws_a, ws_b = Path(tmp_a), Path(tmp_b) + for ws in (ws_a, ws_b): + _touch(ws / "daily" / DAY / "session.md", "same discussion content") + + # Run the full proactive chain on ws_a only. + reply = _reply(follow_ups=[_topic("主动主题", confidence=0.9, paths=[f"daily/{DAY}/session.md"])]) + await _run_chain(ws_a, [reply]) + + units_a = await dream_units(ws_a) + units_b = await dream_units(ws_b) + assert units_a == units_b + assert len(units_a) == 1 + + asyncio.run(run()) + + +# --------------------------------------------------------------------------- +# F4 short-circuit, budget, timeout +# --------------------------------------------------------------------------- + + +def test_llm_timeout(tmp_path): + """A hanging reply times out, short-circuits the round, and retries next round.""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "fresh material") + wrapper = _AgentWrapper(["never returned"], delay=0.5) + catalog = _Catalog() + _, wrapper, catalog, context, responses = await _run_chain( + ws, + wrapper=wrapper, + catalog=catalog, + extract_kwargs={"llm_timeout_seconds": 0.1}, + ) + assert context.get("proactive_skip")["reason"] == "llm_timeout" + assert wrapper.calls == 1 + assert not _interests(ws).exists() + assert catalog.upserts == [] and catalog.dumps == 0 + assert all(r.success for r in responses) + + asyncio.run(run()) + + +def test_parse_failure_retry_then_empty(tmp_path): + """Unparseable output retries once; still failing yields empty, not error.""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "material") + wrapper = _AgentWrapper(["this is not structured at all", _reply(follow_ups=[], updates=[])]) + _, wrapper, _, context, responses = await _run_chain(ws, wrapper=wrapper) + assert wrapper.calls == 2 + assert all(r.success for r in responses) + assert context.get("proactive")["follow_ups"] == [] + + asyncio.run(run()) + + +def test_parse_double_failure_keeps_material_uncheckpointed(tmp_path): + """Two unusable replies skip without checkpoint; material retries next round (audit #1).""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "material") + wrapper = _AgentWrapper(["garbage one", "garbage two"]) + catalog = _Catalog() + app = ApplicationContext(workspace_dir=str(ws)) + context = RuntimeContext(date=DAY, file_catalog=catalog, file_store=_FileStore(ws), agent_wrapper=wrapper) + extract = ProactiveExtractStep(app_context=app) + resp_extract = await extract(context) + topics = ProactiveTopicsStep(app_context=app) + resp_topics = await topics(context) + finish = ProactiveFinishStep(app_context=app) + resp_finish = await finish(context) + assert all(r.success for r in (resp_extract, resp_topics, resp_finish)) + assert wrapper.calls == 2 + assert context.get("proactive_skip") == {"reason": "extract_parse_failed"} + assert not catalog.upserts # nothing checkpointed; material stays changed + + asyncio.run(run()) + + +def test_parse_extract_reply_schema_gate(): + """Non-empty replies without contract sections count as parse failures (audit #1).""" + assert parse_extract_reply("followups:\n - title: misspelled section\n") == {} + assert parse_extract_reply("foo: 1") == {} + assert parse_extract_reply("```yaml\nfollow_ups: []\n```") == {"follow_ups": []} + assert parse_extract_reply("follow_ups: []\nupdates: []") == {"follow_ups": [], "updates": []} + + +def test_schema_error_reply_retries(tmp_path): + """Non-empty YAML with wrong section names retries once like a parse failure (audit #1).""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "material") + bad = "followups:\n - title: 拼错段名的话题\n reason: because\n" + good = _reply(follow_ups=[_topic("补救话题", paths=[f"daily/{DAY}/session.md"])]) + wrapper = _AgentWrapper([bad, good]) + _, wrapper, _, context, responses = await _run_chain(ws, wrapper=wrapper) + assert wrapper.calls == 2 + assert all(r.success for r in responses) + assert [t["title"] for t in context.get("proactive")["follow_ups"]] == ["补救话题"] + + asyncio.run(run()) + + +def test_pack_paths_total_budget(tmp_path): + """max_total_chars keeps the first file and reports how many were omitted (audit #5).""" + for name in ("a.md", "b.md", "c.md"): + _touch(tmp_path / name, "x" * 100) + packed = pack_paths(tmp_path, ["a.md", "b.md", "c.md"], limit_per_file=60000, max_total_chars=150) + assert "### a.md" in packed + assert "### b.md" not in packed + assert "(omitted 2 file(s)" in packed + + +def test_extract_material_budget(tmp_path): + """Blob packing honors max_total_chars; newest daily material is packed first (audit #5).""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / "2026-08-12" / "big-old.md", "x" * 500) + _touch(ws / "daily" / DAY / "fresh.md", "fresh material") + reply = _reply(follow_ups=[_topic("预算话题", paths=[f"daily/{DAY}/fresh.md"])]) + wrapper = _AgentWrapper([reply]) + await _run_chain(ws, wrapper=wrapper, extract_kwargs={"max_total_chars": 200}) + blob = wrapper.last_inputs[0] + assert "### daily/2026-08-13/fresh.md" in blob # newest day survives the budget + assert "### daily/2026-08-12/big-old.md" not in blob # omitted from the material section + assert "omitted" in blob + + asyncio.run(run()) + + +# --------------------------------------------------------------------------- +# F5 read interface +# --------------------------------------------------------------------------- + + +def test_proactive_backward_compat(tmp_path): + """Default params on a v1 file keep the legacy answer field-for-field.""" + + async def run(): + content = ( + "date: 2026-05-28\n" + "topics:\n" + " - title: Retrieval quality\n" + " reason: Search behavior changed repeatedly.\n" + " evidence: daily/2026-05-28/session.md\n" + ) + _touch(tmp_path / "daily" / "2026-05-28" / "interests.yaml", content) + step = ProactiveStep(app_context=ApplicationContext(workspace_dir=str(tmp_path))) + response = await step( + RuntimeContext(date="2026-05-28", include_content=True, file_store=_FileStore(tmp_path)), + ) + assert response.success is True + assert response.answer == { + "summary": "Read 1 proactive topic(s) from daily/2026-05-28/interests.yaml", + "topics": [ + { + "title": "Retrieval quality", + "reason": "Search behavior changed repeatedly.", + "evidence": "daily/2026-05-28/session.md", + "paths": [], + }, + ], + "content": content, + } + assert response.metadata["topics"] == response.answer["topics"] + assert response.metadata["content"] == content + assert response.metadata["push"] is True # additive keys only + + asyncio.run(run()) + + +def test_proactive_answer_omits_unrequested_content(tmp_path): + """Raw YAML is absent from the primary answer when include_content is false.""" + + async def run(): + _touch(tmp_path / "daily" / DAY / "interests.yaml", "topics:\n - title: Topic\n reason: Reason\n") + step = ProactiveStep(app_context=ApplicationContext(workspace_dir=str(tmp_path))) + + response = await step(RuntimeContext(date=DAY, include_content=False, file_store=_FileStore(tmp_path))) + + assert response.success is True + assert "content" not in response.answer + assert response.answer["topics"][0]["title"] == "Topic" + assert response.metadata["content"] == "" + + asyncio.run(run()) + + +def test_proactive_missing_file_is_an_explicit_skip(tmp_path): + """A missing single-day exposure remains distinguishable without metadata.""" + + async def run(): + step = ProactiveStep(app_context=ApplicationContext(workspace_dir=str(tmp_path))) + response = await step(RuntimeContext(date=DAY, file_store=_FileStore(tmp_path))) + + assert response.success is True + assert response.answer == f"Skipped: interests file not found at daily/{DAY}/interests.yaml" + assert response.metadata["skipped"] is True + + asyncio.run(run()) + + +def test_strict_topic_loading_rejects_lossy_fields(tmp_path): + """Strict legacy loading rejects values that would otherwise be discarded.""" + target = _touch(tmp_path / "interests.yaml", "topics:\n - title: Topic\n reason: Reason\n custom: keep me\n") + + try: + load_yaml_topics(target, strict=True) + except ValueError as exc: + assert "unknown field(s): custom" in str(exc) + else: + raise AssertionError("strict topic loading accepted a lossy field") + + +def test_strict_topic_loading_rejects_invalid_field_types(tmp_path): + """Strict legacy loading rejects list fields that would otherwise be normalized away.""" + target = _touch( + tmp_path / "interests.yaml", + "topics:\n - title: Topic\n reason: Reason\n paths: daily/source.md\n", + ) + + try: + load_yaml_topics(target, strict=True) + except ValueError as exc: + assert "topics[0].paths must be a list of non-empty strings" in str(exc) + else: + raise AssertionError("strict topic loading accepted an invalid paths type") + + +def test_horizon_merge(tmp_path): + """Horizon>1 reads the truth source filtered by evidence recency (v5 R4).""" + + async def run(): + ws = tmp_path + id_x, id_y = topic_id("Topic X"), topic_id("Topic Y") + _write_state( + ws, + open_topics=[ + _state_topic("Topic X", day=DAY, topic_id_value=id_x, confidence=0.9), + _state_topic("Topic Y", day="2026-08-09", topic_id_value=id_y, kind="interest_extend", confidence=0.6), + ], + ) + step = ProactiveStep(app_context=ApplicationContext(workspace_dir=str(ws)), horizon_days=3) + response = await step(RuntimeContext(date=DAY, include_content=False, file_store=_FileStore(ws))) + assert response.success is True + assert [t["title"] for t in response.answer["topics"]] == ["Topic X"] # Y evidence too old + assert response.metadata["path"] == "daily/_proactive.yaml" + + # A wider horizon includes the older topic again. + response2 = await step( + RuntimeContext(date=DAY, include_content=False, file_store=_FileStore(ws), horizon_days=7), + ) + assert {t["title"] for t in response2.answer["topics"]} == {"Topic X", "Topic Y"} + + # Historical reads are bounded on both sides and cannot expose topics + # whose evidence is newer than the requested date. + historical = await step( + RuntimeContext(date="2026-08-10", include_content=False, file_store=_FileStore(ws), horizon_days=7), + ) + assert [t["title"] for t in historical.answer["topics"]] == ["Topic Y"] + + # Empty truth source -> skipped, not an error. + with tempfile.TemporaryDirectory() as tmp: + empty_ws = Path(tmp) + response3 = await step( + RuntimeContext(date=DAY, include_content=False, file_store=_FileStore(empty_ws), horizon_days=3), + ) + assert response3.success is True + assert response3.metadata["skipped"] is True + + asyncio.run(run()) + + +def test_min_confidence(tmp_path): + """min_confidence filters v2 topics; v1 topics fall back to 0.5.""" + + async def run(): + ws = tmp_path + _write_interests_v2( + ws, + DAY, + [ + _state_topic("High topic", confidence=0.7), + _state_topic("Low topic", kind="interest_extend", confidence=0.3), + ], + ) + step = ProactiveStep(app_context=ApplicationContext(workspace_dir=str(ws)), min_confidence=0.5) + response = await step(RuntimeContext(date=DAY, include_content=False, file_store=_FileStore(ws))) + assert [t["title"] for t in response.answer["topics"]] == ["High topic"] + + # Default min_confidence=0.4 sits below the 0.5 fallback but above weak 0.3. + step_default = ProactiveStep(app_context=ApplicationContext(workspace_dir=str(ws))) + default_response = await step_default( + RuntimeContext(date=DAY, include_content=False, file_store=_FileStore(ws)), + ) + assert [t["title"] for t in default_response.answer["topics"]] == ["High topic"] + + with tempfile.TemporaryDirectory() as tmp: + ws_v1 = Path(tmp) + _write_interests_v1(ws_v1, DAY, [_topic("Legacy topic", paths=[f"daily/{DAY}/a.md"])]) + step_v1 = ProactiveStep(app_context=ApplicationContext(workspace_dir=str(ws_v1))) + ok = await step_v1( + RuntimeContext(date=DAY, include_content=False, file_store=_FileStore(ws_v1), min_confidence=0.5), + ) + assert [t["title"] for t in ok.answer["topics"]] == ["Legacy topic"] # fallback 0.5 passes + filtered = await step_v1( + RuntimeContext(date=DAY, include_content=False, file_store=_FileStore(ws_v1), min_confidence=0.6), + ) + assert filtered.answer["topics"] == [] + + asyncio.run(run()) + + +# --------------------------------------------------------------------------- +# M2: extends branch + semantic dedup +# --------------------------------------------------------------------------- + + +def test_extend_from_daily_material(tmp_path): + """Branch B infers extends from daily material; out-of-M paths are dropped.""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "discussion about source attribution") + reply = _reply( + follow_ups=[], + extends=[ + _topic("引用质量评测", confidence=0.5, paths=[f"daily/{DAY}/session.md"]), + _topic("越界扩展", confidence=0.5, paths=["daily/2026-08-01/absent.md"]), + ], + ) + _, wrapper, _, context, _ = await _run_chain(ws, [reply]) + assert "extends" in wrapper.last_inputs[0] # extends section rendered + state = context.get("proactive") + assert len(state["extends"]) == 1 + extend = state["extends"][0] + assert extend["kind"] == "interest_extend" + assert f"daily/{DAY}/session.md" in extend["paths"] + assert state["dropped_missing"] == 1 + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert [t["title"] for t in data["topics"]] == ["引用质量评测"] + + asyncio.run(run()) + + +def test_extends_disabled_m1_shape(): + """extends_enabled=False restores the M1 prompt shape and ignores extends output.""" + + async def run_once(extends_enabled: bool): + with tempfile.TemporaryDirectory() as tmp: + ws = Path(tmp) + _touch(ws / "daily" / DAY / "s1.md", "material") + reply = _reply( + follow_ups=[_topic("Follow Up Topic", confidence=0.9, paths=[f"daily/{DAY}/s1.md"])], + extends=[_topic("Extends Topic", confidence=0.9, paths=[f"daily/{DAY}/s1.md"])], + ) + _, wrapper, _, context, _ = await _run_chain( + ws, + [reply], + extract_kwargs={"extends_enabled": extends_enabled}, + ) + return wrapper, context.get("proactive") + + async def run(): + wrapper_on, state_on = await run_once(True) + assert "extends:" in wrapper_on.last_system_prompts[0] # M2 default renders branch B + assert [t["title"] for t in state_on["extends"]] == ["Extends Topic"] + + wrapper_off, state_off = await run_once(False) + system_off = wrapper_off.last_system_prompts[0] + assert "extends" not in system_off.replace("interest_extend", "") # M1 prompt shape + assert state_off["extends"] == [] # extends entries in the reply are ignored + assert [t["title"] for t in state_off["topics_out"]] == ["Follow Up Topic"] + + asyncio.run(run()) + + +def test_semantic_dedup(tmp_path): + """With embeddings, similarity below known_threshold is kept; no LLM tier (v5 R2).""" + + async def run(): + ws = tmp_path + _write_state(ws, open_topics=[_state_topic("Gray Comparison", kind="interest_extend")]) + _touch(ws / "daily" / DAY / "session.md", "material") + reply = _reply( + follow_ups=[ + _topic("Candidate Topic", confidence=0.8, paths=[f"daily/{DAY}/session.md"]), + ], + ) + fake = _FakeEmbedding( + { + # v5.2 scheme: candidate and known sides both embed "title + CJK period + reason" + "Candidate Topic。because": [1.0, 0.0], + "Gray Comparison。seeded": [0.8, 0.6], # cos = 0.800 < 0.85 -> kept, no second tier + }, + ) + _, wrapper, _, context, _ = await _run_chain(ws, [reply], extra_ctx={"as_embedding": fake}) + state = context.get("proactive") + assert state["dropped_known"] == 0 + assert state["dropped_duplicate"] == 0 + assert [c["title"] for c in state["candidates"]] == ["Candidate Topic"] + assert fake.calls >= 1 + assert wrapper.calls == 1 # topics step is pure computation now + + asyncio.run(run()) + + +def test_known_drop(tmp_path): + """sim >= known_threshold drops the candidate as already known.""" + + async def run(): + ws = tmp_path + _write_state(ws, open_topics=[_state_topic("Known Topic", kind="interest_extend")]) + _touch(ws / "daily" / DAY / "session.md", "material") + reply = _reply( + follow_ups=[_topic("Fresh Candidate", confidence=0.8, paths=[f"daily/{DAY}/session.md"])], + ) + fake = _FakeEmbedding( + { + "Fresh Candidate。because": [1.0, 0.0], + "Known Topic。seeded": [1.0, 0.0], # identical -> sim 1.0 >= 0.85 + }, + ) + _, _, _, context, _ = await _run_chain(ws, [reply], extra_ctx={"as_embedding": fake}) + state = context.get("proactive") + assert state["dropped_known"] == 1 + assert state["candidates"] == [] + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert [t["title"] for t in data["topics"]] == ["Known Topic"] + + asyncio.run(run()) + + +# --------------------------------------------------------------------------- +# F2.5/F2.6 plan + agenda steps +# --------------------------------------------------------------------------- + + +def _plan_reply(cards: list[dict]) -> str: + body = yaml.safe_dump({"cards": cards}, allow_unicode=True, sort_keys=False) + return f"```yaml\n{body}```" + + +def _agenda_reply(agenda: list[dict], suppressed: list[dict] | None = None) -> str: + doc = {"agenda": agenda} + if suppressed is not None: + doc["suppressed"] = suppressed + body = yaml.safe_dump(doc, allow_unicode=True, sort_keys=False) + return f"```yaml\n{body}```" + + +def _card(topic_id_value, index=0, scenario="resume_task"): + return { + "topic_id": topic_id_value, + "scenario_type": scenario, + "opener": f"对了，上次那件事还差一步（{index}），要不要现在动起来？", + "next_action": f"执行第 {index} 步", + "preconditions": [], + "delivery": "in_conversation", + } + + +async def _run_full_chain( + ws, + replies=None, + *, + wrapper=None, + catalog=None, + app=None, + extract_kwargs=None, + topics_kwargs=None, + plan_kwargs=None, + agenda_kwargs=None, + extra_ctx=None, + context=None, +): + app = app or ApplicationContext(workspace_dir=str(ws)) + wrapper = wrapper or _AgentWrapper(replies or []) + catalog = catalog or _Catalog() + context = context or RuntimeContext( + date=DAY, + file_catalog=catalog, + file_store=_FileStore(ws), + agent_wrapper=wrapper, + **(extra_ctx or {}), + ) + extract = ProactiveExtractStep(app_context=app, **(extract_kwargs or {})) + resp_extract = await extract(context) + topics = ProactiveTopicsStep(app_context=app, **(topics_kwargs or {})) + resp_topics = await topics(context) + plan = ProactivePlanStep(app_context=app, **(plan_kwargs or {})) + resp_plan = await plan(context) + agenda = ProactiveAgendaStep(app_context=app, **(agenda_kwargs or {})) + resp_agenda = await agenda(context) + finish = ProactiveFinishStep(app_context=app) + resp_finish = await finish(context) + return app, wrapper, catalog, context, (resp_extract, resp_topics, resp_plan, resp_agenda, resp_finish) + + +def test_plan_agenda_chain_e2e(tmp_path): + """extract -> topics -> plan -> agenda -> finish writes the enriched v2 file.""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "we discussed the eval plan") + extract_reply = _reply( + follow_ups=[_topic("记忆检索的可解释性评估", confidence=0.9, paths=[f"daily/{DAY}/session.md"])], + extends=[_topic("多跳 RAG 综述", confidence=0.5, paths=[f"daily/{DAY}/session.md"])], + ) + fid, eid = topic_id("记忆检索的可解释性评估"), topic_id("多跳 RAG 综述") + plan_reply = _plan_reply( + [ + { + "topic_id": fid, + "scenario_type": "resume_task", + "opener": "对了，上次那个评估还差一步，要不要现在跑起来？", + "next_action": "运行评估脚本", + "preconditions": ["数据集已就绪"], + "delivery": "agenda_item", + }, + _card(eid, 1, scenario="explore_interest"), + ], + ) + agenda_text = _agenda_reply( + [{"topic_id": fid, "order_reason": "卡点已解除，今天最适合收尾"}], + [{"topic_id": eid, "reason": f"merged into {fid}"}], + ) + _, wrapper, _, context, resps = await _run_full_chain(ws, [extract_reply, plan_reply, agenda_text]) + assert all(r.success for r in resps) + assert wrapper.calls == 3 + state = context.get("proactive") + assert state["llm_calls"] == 1 # extract only + assert state["plan_llm_calls"] == 2 # plan + agenda + assert state["push"] is True + + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert data["version"] == 2 + assert data["push"] is True + assert len(data["agenda"]) == 1 + item = data["agenda"][0] + assert item["topic_id"] == fid + assert item["title"] == "记忆检索的可解释性评估" + assert item["scenario_type"] == "resume_task" + assert item["opener"].startswith("对了") + assert item["next_action"] == "运行评估脚本" + assert item["preconditions"] == ["数据集已就绪"] + assert item["delivery"] == "agenda_item" + assert item["linked_memory"] == [f"daily/{DAY}/session.md"] + assert item["order_reason"] == "卡点已解除，今天最适合收尾" + assert data["suppressed"] == [ + {"topic_id": eid, "title": "多跳 RAG 综述", "reason": f"merged into {fid}"}, + ] + assert {t["id"] for t in data["topics"]} == {fid, eid} # topics view intact + + asyncio.run(run()) + + +def test_plan_agenda_fallback_without_llm(tmp_path): + """Unusable plan/agenda replies degrade to fallback cards + deterministic agenda.""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "material") + extract_reply = _reply( + follow_ups=[ + _topic("任务一", confidence=0.9, paths=[f"daily/{DAY}/session.md"]), + _topic("任务二", confidence=0.7, paths=[f"daily/{DAY}/session.md"]), + ], + ) + # Only the extract reply is scripted; plan/agenda receive "" and must degrade. + _, wrapper, _, context, resps = await _run_full_chain(ws, [extract_reply]) + assert all(r.success for r in resps) + assert wrapper.calls == 3 + state = context.get("proactive") + assert state["plan_llm_calls"] == 2 # attempted but unusable + assert len(state["scenario_cards"]) == 2 + + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert data["push"] is True + assert [i["title"] for i in data["agenda"]] == ["任务一", "任务二"] + assert all(i["order_reason"] == "deterministic fallback: freshness/confidence order" for i in data["agenda"]) + assert data["suppressed"] == [] + assert all(i["scenario_type"] == "resume_task" for i in data["agenda"]) + assert all(i["opener"] for i in data["agenda"]) + + asyncio.run(run()) + + +def test_agenda_no_candidates_keeps_topics_render(tmp_path): + """Without push candidates plan/agenda skip and the topics-only render stays.""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "material") + extract_reply = _reply( + follow_ups=[_topic("低置信话题", confidence=0.3, paths=[f"daily/{DAY}/session.md"])], + ) + _, wrapper, _, context, resps = await _run_full_chain(ws, [extract_reply]) + assert all(r.success for r in resps) + assert wrapper.calls == 1 # extract only; plan/agenda made zero LLM calls + state = context.get("proactive") + assert state["push"] is False + assert state["push_candidates"] == [] + assert state["scenario_cards"] == [] + assert state["agenda"] == [] + assert state["suppressed"] == [] + + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert data["push"] is False + assert "agenda" not in data and "suppressed" not in data + + asyncio.run(run()) + + +def test_agenda_single_candidate_skips_llm(tmp_path): + """Exactly one candidate is auto-agenda without an agenda LLM call.""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "material") + extract_reply = _reply( + follow_ups=[_topic("唯一候选", confidence=0.9, paths=[f"daily/{DAY}/session.md"])], + ) + fid = topic_id("唯一候选") + plan_reply = _plan_reply([_card(fid, scenario="answer_pending")]) + _, wrapper, _, context, _ = await _run_full_chain(ws, [extract_reply, plan_reply]) + assert wrapper.calls == 2 # extract + plan only + state = context.get("proactive") + assert state["plan_llm_calls"] == 1 + + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert len(data["agenda"]) == 1 + assert data["agenda"][0]["topic_id"] == fid + assert data["agenda"][0]["order_reason"] == "single push candidate; auto-agenda without LLM" + assert data["suppressed"] == [] + + asyncio.run(run()) + + +def test_agenda_merge_and_full_accounting(tmp_path): + """LLM merges two entrances of one matter; every candidate is accounted for.""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "material") + titles = ["GPU 排期确认", "跑 benchmark 的 GPU 档期", "多跳引用可信度"] + extract_reply = _reply( + follow_ups=[ + _topic(titles[0], confidence=0.9, paths=[f"daily/{DAY}/session.md"]), + _topic(titles[1], confidence=0.7, paths=[f"daily/{DAY}/session.md"]), + _topic(titles[2], confidence=0.9, paths=[f"daily/{DAY}/session.md"]), + ], + ) + ids = [topic_id(t) for t in titles] + plan_reply = _plan_reply([_card(tid, i) for i, tid in enumerate(ids)]) + agenda_text = _agenda_reply( + [{"topic_id": ids[0], "order_reason": "主入口"}, {"topic_id": ids[2], "order_reason": "独立事项"}], + [{"topic_id": ids[1], "reason": f"merged into {ids[0]}"}], + ) + _, _, _, context, _ = await _run_full_chain(ws, [extract_reply, plan_reply, agenda_text]) + state = context.get("proactive") + assert state["push"] is True + + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert [i["topic_id"] for i in data["agenda"]] == [ids[0], ids[2]] + assert data["suppressed"] == [ + {"topic_id": ids[1], "title": titles[1], "reason": f"merged into {ids[0]}"}, + ] + accounted = {i["topic_id"] for i in data["agenda"]} | {x["topic_id"] for x in data["suppressed"]} + assert accounted == set(ids) + + asyncio.run(run()) + + +def test_agenda_invalid_ids_and_missing_accounting(tmp_path): + """Bogus agenda ids are dropped; candidates the LLM forgot are still suppressed.""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "material") + titles = ["候选甲", "候选乙"] + extract_reply = _reply( + follow_ups=[ + _topic(titles[0], confidence=0.9, paths=[f"daily/{DAY}/session.md"]), + _topic(titles[1], confidence=0.7, paths=[f"daily/{DAY}/session.md"]), + ], + ) + ids = [topic_id(t) for t in titles] + plan_reply = _plan_reply([_card(tid, i) for i, tid in enumerate(ids)]) + agenda_text = _agenda_reply( + [{"topic_id": "nonexistent01", "order_reason": "bogus"}, {"topic_id": ids[0], "order_reason": "real"}], + [], + ) + _, _, _, _, _ = await _run_full_chain(ws, [extract_reply, plan_reply, agenda_text]) + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert [i["topic_id"] for i in data["agenda"]] == [ids[0]] + assert data["suppressed"] == [ + {"topic_id": ids[1], "title": titles[1], "reason": "not selected for today's agenda"}, + ] + + asyncio.run(run()) + + +def test_plan_over_budget_suppressed(tmp_path): + """Candidates beyond max_plan_topics stay card-less and are suppressed with reason.""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "session.md", "material") + titles = ["高置信一", "高置信二", "高置信三"] + extract_reply = _reply( + follow_ups=[ + _topic(titles[0], confidence=0.9, paths=[f"daily/{DAY}/session.md"]), + _topic(titles[1], confidence=0.8, paths=[f"daily/{DAY}/session.md"]), + _topic(titles[2], confidence=0.7, paths=[f"daily/{DAY}/session.md"]), + ], + ) + ids = [topic_id(t) for t in titles] + # deterministic sort order: confidence desc -> ids[0], ids[1] selected; ids[2] over budget + plan_reply = _plan_reply([_card(ids[0], 0), _card(ids[1], 1)]) + agenda_text = _agenda_reply( + [{"topic_id": ids[0], "order_reason": "first"}, {"topic_id": ids[1], "order_reason": "second"}], + [], + ) + _, _, _, context, _ = await _run_full_chain( + ws, + [extract_reply, plan_reply, agenda_text], + plan_kwargs={"max_plan_topics": 2}, + ) + state = context.get("proactive") + assert len(state["scenario_cards"]) == 2 + data = yaml.safe_load(_interests(ws).read_text(encoding="utf-8")) + assert [i["topic_id"] for i in data["agenda"]] == [ids[0], ids[1]] + assert data["suppressed"] == [ + { + "topic_id": ids[2], + "title": titles[2], + "reason": "over plan budget: not expanded into a scenario card", + }, + ] + + asyncio.run(run()) + + +def test_read_side_agenda_passthrough(tmp_path): + """F5 read passes the agenda through, filtering resolved/unknown/low-confidence ids.""" + + async def run(): + ws = tmp_path + id_a, id_b = topic_id("议程主题"), topic_id("已解决主题") + payload = { + "version": 2, + "date": DAY, + "generated_at": f"{DAY}T09:00:00", + "push": True, + "topics": [ + _state_topic("议程主题", topic_id_value=id_a, confidence=0.9), + _state_topic("已解决主题", topic_id_value=id_b, confidence=0.9), + ], + "agenda": [ + {"topic_id": id_a, "opener": "对了，那件事可以动了"}, + {"topic_id": id_b, "opener": "resolved, must be filtered"}, + {"topic_id": "unknown000000", "opener": "not in kept"}, + ], + "suppressed": [], + } + _touch(_interests(ws, DAY), yaml.safe_dump(payload, allow_unicode=True, sort_keys=False)) + _write_state(ws, resolved=[{"id": id_b, "title": "已解决主题", "resolved_at": DAY, "evidence": ""}]) + + step = ProactiveStep(app_context=ApplicationContext(workspace_dir=str(ws))) + response = await step(RuntimeContext(date=DAY, include_content=False, file_store=_FileStore(ws))) + assert [t["title"] for t in response.answer["topics"]] == ["议程主题"] + assert response.answer["agenda"] == [{"topic_id": id_a, "opener": "对了，那件事可以动了"}] + + asyncio.run(run()) + + +# --------------------------------------------------------------------------- +# F2.7 personal profile (digest/personal) + configured daily_dir +# --------------------------------------------------------------------------- + + +def test_strip_frontmatter_and_profile_loader(tmp_path): + """digest/personal/*.md -> profile block; per-file budget; fallbacks.""" + ws = tmp_path + personal = ws / "digest" / "personal" + personal.mkdir(parents=True) + (personal / "user-profile.md").write_text( + "---\ndescription: 用户张三的身份画像\nname: user-profile\n---\n\n用户张三，偏好西瓜。\n", + encoding="utf-8", + ) + (personal / "user-position.md").write_text( + "---\ndescription: 持仓约束\nname: user-position\n---\n\n持有小米，浮亏较深。\n", + encoding="utf-8", + ) + + meta, body = strip_frontmatter((personal / "user-profile.md").read_text(encoding="utf-8")) + assert meta["name"] == "user-profile" + assert body == "用户张三，偏好西瓜。" + + block = load_personal_profile_block(ws, "digest", 4000) + assert "### user-profile" in block and "### user-position" in block + assert "用户张三的身份画像" in block and "偏好西瓜" in block + assert "持仓约束" in block + + tiny = load_personal_profile_block(ws, "digest", 300) + assert len(tiny) <= 300 + + # absent personal dir -> legacy single-file fallback -> sentinel + empty_ws = tmp_path / "empty" + empty_ws.mkdir() + assert load_personal_profile_block(empty_ws, "digest", 1000) == "(no user profile)" + (empty_ws / "profile.md").write_text("legacy profile", encoding="utf-8") + assert load_personal_profile_block(empty_ws, "digest", 1000, "profile.md") == "legacy profile" + assert load_personal_profile_block(empty_ws, "digest", 0, "profile.md") == "(no user profile)" + + +def test_chain_with_configured_daily_dir(tmp_path): + """The full chain honours daily_dir=memory (paths, truth source, prompts).""" + + async def run(): + ws = tmp_path + _touch(ws / "memory" / DAY / "session.md", "小米财报讨论") + _touch( + ws / "digest" / "personal" / "user-profile.md", + "---\ndescription: 张三画像\nname: user-profile\n---\n\n张三关注小米与中概股。\n", + ) + extract_reply = _reply( + follow_ups=[_topic("小米Q2财报跟进", confidence=0.9, paths=[f"memory/{DAY}/session.md"])], + ) + fid = topic_id("小米Q2财报跟进") + plan_reply = _plan_reply([_card(fid)]) + agenda_text = _agenda_reply([{"topic_id": fid, "order_reason": "only candidate"}], []) + + app = ApplicationContext(workspace_dir=str(ws), daily_dir="memory", digest_dir="digest") + wrapper = _AgentWrapper([extract_reply, plan_reply, agenda_text]) + catalog = _Catalog() + context = RuntimeContext(date=DAY, file_catalog=catalog, file_store=_FileStore(ws), agent_wrapper=wrapper) + extract = ProactiveExtractStep(app_context=app) + resp_extract = await extract(context) + topics = ProactiveTopicsStep(app_context=app) + resp_topics = await topics(context) + plan = ProactivePlanStep(app_context=app) + resp_plan = await plan(context) + agenda = ProactiveAgendaStep(app_context=app) + resp_agenda = await agenda(context) + finish = ProactiveFinishStep(app_context=app) + resp_finish = await finish(context) + assert all(r.success for r in (resp_extract, resp_topics, resp_plan, resp_agenda, resp_finish)) + + state = context.get("proactive") + assert state["daily_dir"] == "memory" + assert state["changed_paths"] == [f"memory/{DAY}/session.md"] + # truth source + exposure land under memory/, not daily/ + assert (ws / "memory" / "_proactive.yaml").is_file() + data = yaml.safe_load((ws / "memory" / DAY / "interests.yaml").read_text(encoding="utf-8")) + assert data["push"] is True + assert [t["id"] for t in data["topics"]] == [fid] + assert not (ws / "daily").exists() + # profile + configured daily dir reached the prompts + assert any("张三画像" in inp or "张三关注小米" in inp for inp in wrapper.last_inputs) + assert any(f"memory/{DAY}" in inp for inp in wrapper.last_inputs) + assert any("memory/" in sp for sp in wrapper.last_system_prompts) + + asyncio.run(run()) + + +def test_finish_defers_paths_modified_during_round(tmp_path): + """Files edited while the LLM calls run are left for the next round (audit 3).""" + + async def run(): + ws = tmp_path + _touch(ws / "daily" / DAY / "a.md", "first note") + note_b = _touch(ws / "daily" / DAY / "b.md", "second note") + reply = _reply(follow_ups=[], extends=[]) + app = ApplicationContext(workspace_dir=str(ws)) + wrapper = _AgentWrapper([reply]) + catalog = _Catalog() + context = RuntimeContext(date=DAY, file_catalog=catalog, file_store=_FileStore(ws), agent_wrapper=wrapper) + + resp_extract = await ProactiveExtractStep(app_context=app)(context) + assert resp_extract.success is True + state = context.get("proactive") + assert set(state["changed_paths"]) == {f"daily/{DAY}/a.md", f"daily/{DAY}/b.md"} + assert state["changed_mtimes"][f"daily/{DAY}/b.md"] == note_b.stat().st_mtime + + # The user edits b.md while plan/agenda LLM calls are still running. + note_b.write_text("second note, edited mid-round", encoding="utf-8") + future = time.time() + 5 + os.utime(note_b, (future, future)) + + resp_finish = await ProactiveFinishStep(app_context=app)(context) + assert resp_finish.success is True + final_state = context.get("proactive") + assert final_state["checkpoint_paths"] == [f"daily/{DAY}/a.md"] + assert "deferred 1" in resp_finish.answer + assert {n.path for n in catalog.upserts} == {f"daily/{DAY}/a.md"} + + asyncio.run(run()) + + +def test_read_single_corrupt_yaml_is_read_only_failure(tmp_path): + """Corrupt interests.yaml: the read fails loudly and never moves the file (audit 2).""" + + async def run(): + ws = tmp_path + corrupt = "topics: [unclosed" + path = _touch(ws / "daily" / DAY / "interests.yaml", corrupt) + step = ProactiveStep(app_context=ApplicationContext(workspace_dir=str(ws))) + response = await step(RuntimeContext(date=DAY, file_store=_FileStore(ws))) + assert response.success is False + assert response.metadata.get("error") + assert path.is_file() + assert path.read_text(encoding="utf-8") == corrupt + assert not list(path.parent.glob("interests.corrupt-*")) + + asyncio.run(run()) + + +def test_carry_forward_expiry_boundary_consistent(tmp_path): + """Age == carry_forward_days is over-age in both prompt loading and trim (audit 9).""" + + async def run(): + ws = tmp_path + edge_day = (dt.date.fromisoformat(DAY) - dt.timedelta(days=14)).isoformat() + _write_state(ws, open_topics=[_state_topic("Edge Topic", day=edge_day)]) + state_file, _ = load_state(ws) + carry_all, _ = await load_carry_forward(ws, state_file, DAY, 14, 20) + assert carry_all == [] + + trim_state_file(state_file, DAY, 14) + assert state_file.open_topics == [] + + asyncio.run(run())