From bb354cc5808d76efc1e88bcc96aaec556855b548 Mon Sep 17 00:00:00 2001 From: Sen Huang <48879559+ployts@users.noreply.github.com> Date: Mon, 25 May 2026 17:52:51 +0800 Subject: [PATCH] refactor(steps): reorganize step modules and remove demo steps (#255) * feat(config): add comprehensive job definitions for vault operations - Add utility jobs like version, search, traverse, list, read, stat - Include file operations like move, delete, upload, download - Add daily workspace management jobs: daily_list, daily_resolve, daily_reindex - Update descriptions to reflect vault-based operations instead of working_dir - Add proper section headers and documentation for each job category refactor(steps): reorganize step modules and remove demo steps - Move steps into categorized packages: common, crud, frontmatter, daily, jobs - Remove demo steps (DemoEchoStep1, DemoEchoStep2, StreamDemoStep1, StreamDemoStep2) - Add new steps: InitStep for vault initialization, TraverseStep for graph traversal - Update __init__.py to auto-import all step modules - Organize imports by functionality (common, CRUD operations, frontmatter, daily) feat(vault): implement vault-centric file operations and configuration - Change default config to use vault_dir instead of working_dir - Add environment variable support for embedding configuration - Implement file watcher with lite backend for daily/digest directories - Update search step to use 'name' instead of 'title' from frontmatter - Create ResourceEntry schema for tracking uploaded assets docs(steps): add comprehensive documentation for all step categories - Document file-I/O split by blast radius (crud vs frontmatter packages) - Add detailed descriptions for each step category and functionality - Explain the purpose and usage patterns for different types of file operations - Provide clear parameter documentation for all new job configurations * fix(config): correct vault directory path and remove unused job configurations - Fix vault_dir from 'vaultd' to 'vault' in default configuration - Remove deprecated traverse and list job configurations - Remove unused tag tooling configurations - Remove background watch_file job configuration refactor(steps): remove unused jobs module import - Comment out jobs module import in steps/__init__.py - This removes unused synchronizer and digester step registrations refactor(tests): update import path and add pylint directive - Update ResourceEntry import from reme4.schema to reme4.schema.resource_meta - Add pylint disable directive for unused argument in test datetime mocks * efactor(steps): remove unused modules from __all__ - Remove "background" module from __all__ list - Remove "jobs" module from __all__ list - These modules were no longer being used in the steps package * feat(config): update vault directory structure and remove file watcher - Change vault_dir reference from ./vault to ./vault in CLI example - Add daily_dir, digest_dir, and resource_dir configuration options - Remove file_watcher component configuration as it's no longer needed - Update comment to reflect correct module name (reme4vault) refactor(steps): add background step and remove deprecated init step - Import and register background step module - Remove deprecated InitStep from common steps - Update __all__ export list to include background step refactor(reindex): improve reindex step to scan vault directly - Update docstring to reflect vault scanning instead of watcher sync - Replace file watcher stop/start logic with direct vault path walking - Add support for suffix filtering during reindex operation - Use index_changes job to process found files refactor(wikilink_utils): enhance inbound source lookup with link scope - Import LinkScopeEnum for proper type handling - Update get_inlinks call to use ALL scope for virtual targets - Improve documentation for reverse-index lookup behavior test(refactor): clean up test suite removing deprecated functionality - Remove test_init_job and test_demo_job unit tests - Update help job assertion to check for literal command format - Change test directory from .reme to vault in CRUD tests - Remove init and demo job calls from integration test BREAKING CHANGE: Removes file_watcher component and init step * style(steps): fix import formatting in __init__.py Add proper spacing in the background module import statement to maintain consistent code style and readability. * refactor(config): change default vault directory from vault to .reme Default dev config now points vault_dir at ./.reme so `python -m reme4 start` can be run from the repo root and exercise the full atomic-tool surface against the seeded test data. BREAKING CHANGE: The default vault directory has been changed from 'vault' to '.reme' in the configuration. * docs(reme4_report): fix markdown formatting and remove extra content * refactor(file_parser): delegate wikilink extraction to WikilinkHandler * fix(search): handle empty query case gracefully - Replace assertion with conditional check for empty query - Set response success to false when query is empty - Return error message instead of throwing assertion error - Maintain existing validation for other parameters --- docs4/reme4_report.md | 30 +- .../components/file_graph/neo4j_file_graph.py | 2 - .../file_parser/linked_file_parser.py | 125 +- reme4/config/default.yaml | 384 +++- reme4/steps/__init__.py | 41 +- reme4/steps/common/__init__.py | 8 +- reme4/steps/common/health_check.py | 3 +- reme4/steps/common/reindex.py | 27 +- reme4/steps/common/search.py | 21 +- reme4/steps/common/traverse.py | 154 ++ reme4/steps/crud/__init__.py | 39 +- reme4/steps/crud/delete.py | 133 ++ reme4/steps/crud/download.py | 84 + reme4/steps/crud/list.py | 55 + reme4/steps/crud/move.py | 132 ++ reme4/steps/crud/stat.py | 72 + reme4/steps/crud/upload.py | 74 + reme4/steps/crud/upload_resource.py | 447 +++++ reme4/steps/daily/__init__.py | 44 + reme4/steps/daily/_day_index.py | 254 +++ reme4/steps/daily/create.py | 84 + reme4/steps/daily/list.py | 45 + reme4/steps/daily/reindex.py | 55 + reme4/steps/daily/resolve.py | 91 + reme4/steps/frontmatter/__init__.py | 24 + reme4/steps/frontmatter/delete.py | 66 + reme4/steps/frontmatter/read.py | 44 + reme4/steps/frontmatter/update.py | 54 + reme4/steps/graph/__init__.py | 7 + reme4/steps/graph/traverse.py | 119 ++ reme4/utils/common_utils.py | 2 +- reme4/utils/wikilink_handler.py | 387 ++++ tests4/unittest/test_common_steps.py | 253 ++- tests4/unittest/test_crud_steps.py | 1554 +++++++++++++++++ tests4/unittest/test_daily_steps.py | 701 ++++++++ tests4/unittest/test_resource_steps.py | 605 +++++++ tests4/unittest/test_wikilink_utils.py | 399 +++++ 37 files changed, 6377 insertions(+), 242 deletions(-) create mode 100644 reme4/steps/common/traverse.py create mode 100644 reme4/steps/crud/delete.py create mode 100644 reme4/steps/crud/download.py create mode 100644 reme4/steps/crud/list.py create mode 100644 reme4/steps/crud/move.py create mode 100644 reme4/steps/crud/stat.py create mode 100644 reme4/steps/crud/upload.py create mode 100644 reme4/steps/crud/upload_resource.py create mode 100644 reme4/steps/daily/__init__.py create mode 100644 reme4/steps/daily/_day_index.py create mode 100644 reme4/steps/daily/create.py create mode 100644 reme4/steps/daily/list.py create mode 100644 reme4/steps/daily/reindex.py create mode 100644 reme4/steps/daily/resolve.py create mode 100644 reme4/steps/frontmatter/__init__.py create mode 100644 reme4/steps/frontmatter/delete.py create mode 100644 reme4/steps/frontmatter/read.py create mode 100644 reme4/steps/frontmatter/update.py create mode 100644 reme4/steps/graph/__init__.py create mode 100644 reme4/steps/graph/traverse.py create mode 100644 reme4/utils/wikilink_handler.py create mode 100644 tests4/unittest/test_crud_steps.py create mode 100644 tests4/unittest/test_daily_steps.py create mode 100644 tests4/unittest/test_resource_steps.py create mode 100644 tests4/unittest/test_wikilink_utils.py diff --git a/docs4/reme4_report.md b/docs4/reme4_report.md index 1c80e97a..b16ca5a9 100644 --- a/docs4/reme4_report.md +++ b/docs4/reme4_report.md @@ -154,21 +154,21 @@ ReMe 没有发明新格式，而是完全复用 Obsidian 生态的约定： - **YAML front matter**：标题、标签、描述、自定义字段。 - ```markdown - --- - title: 光伏产业链研究 - description: 从硅料到组件的全链条梳理 - tags: [新能源, 光伏, 产业链] - parent: 新能源 - author: 张三 - updated: 2026-05-19 - --- +```markdown +--- +title: 光伏产业链研究 +description: 从硅料到组件的全链条梳理 +tags: [新能源, 光伏, 产业链] +parent: 新能源 +author: 张三 +updated。: 2026-05-19 +--- - # 正文从这里开始 - ``` +# 正文从这里开始 - `title` / `description` / `tags` 是约定字段（参见 `reme4/schema/file_front_matter.py`），其余键值对作为 extras - 全部保留，可被检索和图索引消费。 +`title` / `description` / `tags` 是约定字段（参见 `reme4/schema/file_front_matter.py`），其余键值对作为 extras +全部保留，可被检索和图索引消费。 +``` - **4 种 wikilink 写法**： - `[[X]]`：标准链接 @@ -180,9 +180,7 @@ ReMe 没有发明新格式，而是完全复用 Obsidian 生态的约定： - 标准 `[text](xxx.md)` 链接也会被识别为图边。 **意义**：用户的知识库可以直接用 Obsidian 打开做可视化浏览，可以用 Obsidian 插件做扩展。ReMe 不是替代 Obsidian，而是**给 -Obsidian 加上一个会自己写笔记的 Agent**。 - -``` +Obsidian 加上一个会自己写笔记的 Agent** ### 4.2 比 RAG 更聪明的切片 diff --git a/reme4/components/file_graph/neo4j_file_graph.py b/reme4/components/file_graph/neo4j_file_graph.py index f580c9ed..828d2776 100644 --- a/reme4/components/file_graph/neo4j_file_graph.py +++ b/reme4/components/file_graph/neo4j_file_graph.py @@ -32,8 +32,6 @@ Conditional dependency: the ``neo4j`` driver loads lazily; the import error fires at ``_start`` (boot), not at first call. """ -from __future__ import annotations - import json from typing import Any diff --git a/reme4/components/file_parser/linked_file_parser.py b/reme4/components/file_parser/linked_file_parser.py index 2386e4b5..6fa46b3d 100644 --- a/reme4/components/file_parser/linked_file_parser.py +++ b/reme4/components/file_parser/linked_file_parser.py @@ -8,23 +8,12 @@ Pipeline: build mistletoe AST → ``MdNode`` tree (sections nest by heading level) → recursive chunk (try whole subtree; on overflow walk children — body siblings pack as a run, subsections recurse). Leaf blocks (table / code / list / paragraph) split on internal boundaries -and each piece is annotated ``[Part X/N]``. Wikilinks in the body are -extracted as graph edges, with optional Dataview-style typed predicates -(line-level ``predicate:: [[X]]`` or inline-bracketed ``[predicate:: [[X]]]``). - -Wikilink convention. Targets are taken **literally** — ``[[X]]`` -becomes ``target_path="X"`` with no implicit ``.md``, no short-form -basename search, no folder-note expansion. The recommended form is a -full path relative to the vault with extension, e.g. -``[[digest/alice/alice.md]]``. Anything else (``[[Alice]]``, -``[[digest/alice/alice]]``) is also stored verbatim and will be -flagged by ``lint:dangling`` because no node lives at that literal -path — the parser does no validation, lint is the contract enforcer. +and each piece is annotated ``[Part X/N]``. Wikilink extraction is +delegated to :class:`reme4.utils.wikilink_handler.WikilinkHandler` — +the single source of truth for ``[[...]]`` syntax (including +Dataview-style typed predicates). """ -from __future__ import annotations - -import re from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -36,112 +25,10 @@ from .base_file_parser import BaseFileParser from ..component_registry import R from ...schema import ( FileChunk, - FileLink, FileFrontMatter, FileNode, ) - - -# -- Wikilink extraction -------------------------------------------------- - - -_WIKILINK_RE = re.compile( - r""" - (?:!)? - \[\[ - (?P[^\]\|\#\n]+?) - (?:\#(?P[^\]\|\n]+))? - (?:\|[^\]\n]+)? - \]\] - """, - re.VERBOSE, -) - -_DATAVIEW_LINE_RE = re.compile( - r"^[ \t]*(?:[-*+][ \t]+)?(?P[A-Za-z][A-Za-z0-9_]*)\s*::\s*(?P.+?)\s*$", - re.MULTILINE, -) - -_INLINE_FIELD_OPEN_RE = re.compile(r"\[(?P[A-Za-z][A-Za-z0-9_]*)\s*::\s*") - - -def _iter_inline_fields(text: str) -> list[tuple[int, int, str]]: - """Find inline-bracketed ``[predicate:: …]`` field spans by depth scan.""" - out: list[tuple[int, int, str]] = [] - for m in _INLINE_FIELD_OPEN_RE.finditer(text): - depth = 1 - i = m.end() - n = len(text) - while i < n: - c = text[i] - if c == "\n": - break - if c == "[": - depth += 1 - elif c == "]": - depth -= 1 - if depth == 0: - out.append((m.start(), i + 1, m.group("predicate"))) - break - i += 1 - return out - - -def _predicate_for( - text: str, - pos: int, - inline_spans: list[tuple[int, int, str]], -) -> str | None: - """Resolve the predicate governing a wikilink at offset ``pos``. - - Precedence: inline-bracketed > line-level Dataview > none. - """ - for field_start, field_end, predicate in inline_spans: - if field_start <= pos < field_end: - return predicate - line_start = text.rfind("\n", 0, pos) + 1 - line_end = text.find("\n", pos) - if line_end == -1: - line_end = len(text) - m = _DATAVIEW_LINE_RE.match(text[line_start:line_end]) - if m and line_start + m.start("value") <= pos: - return m.group("predicate") - return None - - -def _extract_links(text: str, source_path: str) -> list[FileLink]: - """Find every wikilink in ``text`` and emit FileLinks with literal targets. - - No resolution is performed: ``target_path`` is the bracket contents - verbatim. ``lint:dangling`` checks whether the literal target exists - in the graph. Results are deduped by - ``(target_path, predicate, target_anchor)`` preserving order. - """ - if not text: - return [] - inline_spans = _iter_inline_fields(text) - out: list[FileLink] = [] - seen: set[tuple] = set() - for wm in _WIKILINK_RE.finditer(text): - target = wm.group("target").strip() - if not target: - continue - anchor_raw = wm.group("anchor") - anchor = anchor_raw.strip() if anchor_raw else None - predicate = _predicate_for(text, wm.start(), inline_spans) - key = (target, predicate, anchor) - if key in seen: - continue - seen.add(key) - out.append( - FileLink( - source_path=source_path, - target_path=target, - target_anchor=anchor, - predicate=predicate, - ), - ) - return out +from ...utils.wikilink_handler import WikilinkHandler # -- AST node + helpers --------------------------------------------------- @@ -249,7 +136,7 @@ class LinkedFileParser(BaseFileParser): tree = self._build_tree(Document(post.content), renderer) chunks = self._chunk_node(tree, "", "", rel_path, renderer) - links = _extract_links(post.content, rel_path) if post.content else [] + links = WikilinkHandler.extract_links(post.content, rel_path) if post.content else [] node = FileNode( path=rel_path, diff --git a/reme4/config/default.yaml b/reme4/config/default.yaml index b27568aa..6e7aa454 100644 --- a/reme4/config/default.yaml +++ b/reme4/config/default.yaml @@ -1,29 +1,19 @@ -daily_dir: memory - service: backend: http # backend: mcp -jobs: - - backend: base - name: demo - description: "demo job description" - parameters: - type: object - properties: - query: - type: string - description: "query" - min_score: - type: number - description: "min score" - default: 0.5 - required: - - query - steps: - - backend: demo_echo_step1 - - backend: demo_echo_step2 +# Default dev config points vault_dir at ./.reme so `python -m reme4 start` +# can be run from the repo root and exercise the full atomic-tool surface +# against the seeded test data. Override the `vault_dir=` CLI arg. +vault_dir: .reme +daily_dir: daily +digest_dir: digest +resource_dir: resource +jobs: + # ════════════════════════════════════════════════════════════════════ + # UTILITY — service introspection + # ════════════════════════════════════════════════════════════════════ - backend: base name: version description: "return reme4 package version" @@ -87,9 +77,14 @@ jobs: steps: - backend: index_changes_step + # ════════════════════════════════════════════════════════════════════ + # ATOMIC TOOLS — same surface plugins/reme-{service,expert} expose + # ════════════════════════════════════════════════════════════════════ + + # ── Retrieve ─────────────────────────────────────────────────────── - backend: base name: search - description: "hybrid search over file_store: vector + keyword fused via RRF" + description: "Hybrid search over the vault (vector + BM25 fused via RRF). Returns ranked chunks with optional wikilink expansion." parameters: type: object properties: @@ -114,14 +109,58 @@ jobs: max_links_per_direction: 10 - backend: base - name: read - description: "read a markdown file (relative path under working_dir)" + name: traverse + description: "Traverse the file graph from one or more seed paths up to N hops, in the chosen direction. Use to chase wikilink neighborhoods after a search hit." parameters: type: object properties: path: type: string - description: "relative path under the working_dir (no absolute paths); markdown only" + description: "Seed path (relative to the vault)." + depth: + type: integer + description: "Hop limit (default 1 = immediate neighbors)." + default: 1 + direction: + type: string + description: "Edge direction filter: forward / backward / both." + default: both + required: + - path + steps: + - backend: traverse_step + + # ── Read Operations ─────────────────────────────────────────────────────────── + - backend: base + name: list + description: "List files under a path; optionally recursive, with a result cap. Plain directory walker — no frontmatter parsing. Callers that need frontmatter-based filtering should iterate the result and call `frontmatter:read` per candidate." + parameters: + type: object + properties: + path: + type: string + description: "Directory path (relative to the vault). Empty = vault root." + default: "" + recursive: + type: boolean + description: "Recurse into subdirectories." + default: false + limit: + type: integer + description: "Max results." + default: 100 + steps: + - backend: list_step + + - backend: base + name: read + description: "read a markdown file (relative path under the vault)" + parameters: + type: object + properties: + path: + type: string + description: "relative path under the vault (no absolute paths); markdown only" start_line: type: integer description: "Optional, first line to read (1-based, inclusive)" @@ -133,6 +172,35 @@ jobs: steps: - backend: read_step + - backend: base + name: stat + description: "Stat a file under the vault (size, mtime, exists, is_dir, is_file)." + parameters: + type: object + properties: + path: + type: string + description: "Path relative to the vault." + required: + - path + steps: + - backend: stat_step + + - backend: base + name: frontmatter:read + description: "Read a markdown file's parsed YAML frontmatter as a dict (body excluded)." + parameters: + type: object + properties: + path: + type: string + description: "Path relative to the vault." + required: + - path + steps: + - backend: frontmatter:read_step + + # ── Write Operations────────────────────────────────────────────────────────── - backend: base name: write description: >- @@ -143,7 +211,7 @@ jobs: properties: path: type: string - description: "relative path under the working_dir; markdown only." + description: "relative path under the vault; markdown only." name: type: string description: "front matter `name` field; short human-readable title of the file." @@ -169,7 +237,7 @@ jobs: properties: path: type: string - description: "relative path under the working_dir; markdown only." + description: "relative path under the vault; markdown only." old: type: string description: "exact text to find." @@ -192,7 +260,7 @@ jobs: properties: path: type: string - description: "relative path under the working_dir; markdown only." + description: "relative path under the vault; markdown only." content: type: string description: "content to append." @@ -202,28 +270,228 @@ jobs: steps: - backend: append_step - - backend: stream - name: stream_demo - description: "stream demo job: repeat query 10x and stream char-by-char" + - backend: base + name: frontmatter:update + description: "Update YAML frontmatter on an existing markdown file (merge semantics — each entry in `metadata` becomes one frontmatter key; missing keys are inserted, existing keys overwritten). Use for surgical edits to the reserved keys (`name` / `description`) or any caller-defined keys without rewriting the body." parameters: type: object properties: - query: + path: type: string - description: "query to echo" - repeat: - type: integer - description: "number of times to repeat the query" - default: 10 - interval: - type: number - description: "seconds between chunks" - default: 0.1 + description: "Path relative to the vault." + metadata: + type: object + description: "Frontmatter keys to merge — every key/value becomes one frontmatter entry." + additionalProperties: true required: - - query + - path + - metadata steps: - - backend: stream_demo_step1 - - backend: stream_demo_step2 + - backend: frontmatter_update_step + + - backend: base + name: frontmatter:delete + description: "Delete one or more YAML frontmatter keys on an existing file." + parameters: + type: object + properties: + path: + type: string + description: "Path relative to the vault." + keys: + type: array + description: "Frontmatter keys to remove." + items: + type: string + required: + - path + - keys + steps: + - backend: frontmatter_delete_step + + # ── File Operations (relocate / cross-realm) ────────────────────────────── + - backend: base + name: move + description: "Relocate / rename a file within the vault and (by default) fix every inbound wikilink. Use for promoting a draft (daily/ → digest/) or renaming a slug. retarget=true (default) rewrites [[src_path]] references across the vault to [[dst_path]] after the rename — keeping the knowledge graph consistent without a second tool call. Set retarget=false only when intentionally leaving inbound links dangling (e.g. moving a file aside to delete next). For cross-realm transfer use upload / download." + parameters: + type: object + properties: + src_path: + type: string + description: "Source path (relative to the vault)." + dst_path: + type: string + description: "Destination path (relative to the vault) with a directory component." + overwrite: + type: boolean + description: "Overwrite if dst_path exists." + default: false + retarget: + type: boolean + description: "Rewrite inbound wikilinks [[src_path]] → [[dst_path]] after the move (across the vault). Set to false to leave links dangling." + default: true + required: + - src_path + - dst_path + steps: + - backend: move_step + + - backend: base + name: delete + description: "Hard-delete a file or folder under the vault and report every inbound wikilink that pointed at the doomed targets. The delete itself is unconditional — the inbound list (path + count, literal full-path matching only; sources inside the doomed folder are filtered out) is returned so the agent can decide what to do about each surviving reference (edit the citing prose, point it at a replacement, or accept it as dangling)." + parameters: + type: object + properties: + path: + type: string + description: "Path relative to the vault. File or directory." + required: + - path + steps: + - backend: delete_step + + - backend: base + name: upload + description: "Copy a file INTO the vault from the local filesystem. Symmetric counterpart to `download`: source is on the local host, target is vault-relative. `dst_path` is required and must include a directory component so the caller is always explicit about where in the vault the file lands. For the passive-ingest channel-tagged resource bucket (resource// with provenance metadata), use `upload_resource` instead." + parameters: + type: object + properties: + src_path: + type: string + description: "Absolute host filesystem path to the file to copy in." + dst_path: + type: string + description: "Vault-relative destination path (must include a directory component)." + overwrite: + type: boolean + description: "Overwrite if dst_path already exists." + default: false + required: + - src_path + - dst_path + steps: + - backend: upload_step + + - backend: base + name: upload_resource + description: "Land an externally-received asset into resource// (today = local date at call time), alongside a meta.json row capturing provenance and a regenerated .md index view. This is the passive-ingest entry point for external channels (wechat / email / browser / api / ...). Do NOT use it for materials the agent actively fetches or generates inside a daily task — those belong inlined inside the daily note daily//.md. The bucket file name is derived as `____` (callers cannot override); source basenames with path separators, dot segments, or a leading '.' are rejected. Returns {date, name, path}. A duplicate (same channel + same second + same source basename) returns an error — the step never silently dedupes." + parameters: + type: object + properties: + path: + type: string + description: "Source path on the local filesystem; its basename becomes the trailing component of the bucket file name." + channel: + type: string + description: "Inbound channel identifier (wechat / email / browser / api / ...). Lowercase letters / digits / dashes only; must start with a letter or digit." + description: + type: string + description: "Analysis hint for downstream agents: where the asset came from, what kind of content it carries, and how it should be interpreted (skim vs. deep parse, structured extraction vs. summarization, ...). The digester reads this verbatim from meta.json to decide how to process the asset, so write enough detail to drive that decision — not just a title. Multi-line is fine; the .md bullet view flattens for display while meta.json preserves the original." + metadata: + type: object + description: "Optional extras persisted on the meta.json row. `source` (free-form origin within the channel — group name, sender, URL, ...) is conventional; any other keys pass through verbatim. Keys `name`, `channel`, `received_at`, `description` are reserved." + default: {} + required: + - path + - channel + - description + steps: + - backend: upload_resource_step + + - backend: base + name: download + description: "Copy a file OUT of the vault to the local filesystem. Use to hand materials from the vault to local tools (browser, viewer). `dst_path` empty → land in a session-scoped temp file and return the realized path." + parameters: + type: object + properties: + src_path: + type: string + description: "File inside the vault (vault-relative)." + dst_path: + type: string + description: "Absolute host filesystem path to write to. Empty = session-scoped temp file (the realized path is returned)." + default: "" + overwrite: + type: boolean + description: "Overwrite if dst_path already exists." + default: false + required: + - src_path + steps: + - backend: download_step + + # ── Daily Operations(note genesis + day-index rollup) ─────────────────── + # Note body edits use generic file + frontmatter primitives once the + # note exists. + - backend: base + name: daily:list + description: "List the notes under a single day AND rebuild `daily/.md` as a side effect (idempotent — the freshly-rendered note inventory is exactly what callers want to read). Returns {date, notes: [{path, name, description}, ...]} — one row per `daily//.md` note file with vault-relative path / name / description (frontmatter-parsed). Read view of the same rebuild that `daily:reindex` exposes from the write side." + parameters: + type: object + properties: + date: + type: string + description: "ISO date (YYYY-MM-DD). Empty = today." + default: "" + steps: + - backend: daily_list_step + + - backend: base + name: daily:resolve + description: "Ensure the day folder daily// exists and return the vault-relative path daily//.md. Pure path-shape helper — does NOT create the note file (use daily:create or file_write for that). Name must be valid as a filename on Windows (no reserved chars < > : \" / \\ | ? *, no reserved device names CON / PRN / AUX / NUL / COM1-9 / LPT1-9, no trailing '.' or whitespace). Idempotent: when the note file already exists returns {exists: true, message: ...} so the caller knows to read-modify rather than overwrite." + parameters: + type: object + properties: + name: + type: string + description: "Note slug (the .md file's stem). Must satisfy Windows filename rules." + required: + - name + steps: + - backend: daily_resolve_step + + - backend: base + name: daily:create + description: "Create the note file daily//.md with a minimal `name` frontmatter, then refresh the day index daily/.md. Idempotent — if the note file already exists it is left untouched (caller should read-modify rather than overwrite); the index is still refreshed because sibling notes may have changed. Returns {date, slug, path, created, index}." + parameters: + type: object + properties: + slug: + type: string + description: "Note slug (the .md file's stem under daily//)." + body: + type: string + description: "Initial body content of the note file. Empty leaves the file as frontmatter-only." + default: "" + date: + type: string + description: "ISO date (YYYY-MM-DD). Empty = today." + default: "" + name: + type: string + description: "Reserved-field name written to frontmatter. Empty falls back to `slug`." + default: "" + refresh_index: + type: boolean + description: "Refresh daily/.md after the write. Set false for batch flows that will reindex at the end." + default: true + required: + - slug + steps: + - backend: daily_create_step + + - backend: base + name: daily:reindex + description: "Rebuild the day-index page daily/.md from the current set of notes under that date. The day index is a derived artifact; this is the standalone writer — call it once after a batch of note mutations (resolve / write / frontmatter:update), or for historical backfill and drift recovery. Idempotent and safe to re-run. Returns {date, path, created, notes_count} — the write view (was the index page just created? how many notes were swept in?). For the per-note inventory use `daily:list` (which also triggers this rebuild)." + parameters: + type: object + properties: + date: + type: string + description: "ISO date (YYYY-MM-DD). Empty = today." + default: "" + steps: + - backend: daily_reindex_step - backend: background name: watch_file @@ -243,8 +511,10 @@ components: embedding_model: default: - backend: openai - model_name: text-embedding-v4 + backend: ${EMBEDDING_BACKEND:-openai} + api_key: ${EMBEDDING_API_KEY:-} + base_url: ${EMBEDDING_BASE_URL:-https://api.openai.com/v1} + model_name: ${EMBEDDING_MODEL_NAME:-text-embedding-v4} dimensions: 1024 file_graph: @@ -275,8 +545,22 @@ components: file_store: default: backend: local - store_name: default -# embedding_model: default - embedding_model: "" + store_name: local + embedding_model: default keyword_index: default - file_graph: default \ No newline at end of file + file_graph: default + +# as_llm / formatter aren't required for atomic primitives; configure +# only if you'll invoke digester/synchronizer or other LLM-driven +# paths from the dev server. +# as_llm: +# default: +# backend: ${LLM_BACKEND:-openai} +# api_key: ${LLM_API_KEY:-} +# model_name: ${LLM_MODEL_NAME:-gpt-4o-mini} +# client_kwargs: +# base_url: ${LLM_BASE_URL:-https://api.openai.com/v1} +# +# as_llm_formatter: +# default: +# backend: ${LLM_BACKEND:-openai} diff --git a/reme4/steps/__init__.py b/reme4/steps/__init__.py index 715ddc4c..47f486e3 100644 --- a/reme4/steps/__init__.py +++ b/reme4/steps/__init__.py @@ -1,13 +1,46 @@ -"""steps""" +"""steps — registers every BaseStep subclass at import time. -from . import background -from . import common -from . import crud +Each submodule's ``@R.register`` decorators only fire when the module +is imported. Auto-importing them here means any config that names a +step backend (e.g. ``graph_traverse_step``, ``write``, ``digester``) +will find it in the registry without the caller having to remember +which submodule it lives in. + +File-I/O is split by blast radius. The ``crud`` package covers both +opaque-byte ops (list / stat / move / delete / upload / download) and +whole-file text ops (read / write / append / edit) — they share the +same path-resolution helpers, so they live in one package. +``frontmatter`` is the one sliced surface that earns its own RUD +package (YAML is structured data — surgical key edits cannot be safely +emulated with string-substitution on the body). For mid-file body +edits, use ``edit`` (exact string replacement) or do a read + write +round-trip. + +* ``common`` — search / health_check / help / reindex / version / graph_traverse +* ``crud`` — list / stat / move / delete / upload / download / read / write / append / edit +* ``frontmatter`` — markdown frontmatter slice RUD (frontmatter_read_step / update / delete) +* ``daily`` — note genesis / list / day-index reindex +* ``jobs`` — synchronizer / digester (LLM-driven orchestrators) +""" + +from . import common # noqa: F401 -- registers common steps (search, version, graph_traverse, ...) +from . import crud # noqa: F401 -- registers list/stat/upload/download/move/delete/read/write/append/edit +from . import frontmatter # noqa: F401 -- registers frontmatter_read_step/update/delete +from . import ( + daily, +) # noqa: F401 -- registers daily_resolve_step / daily_create_step / daily_list_step / daily_reindex_step +from . import background # noqa: F401 + +# from . import jobs # noqa: F401 -- registers synchronizer / digester from .base_step import BaseStep +from . import graph # noqa: F401 __all__ = [ "background", "common", "crud", + "graph", + "frontmatter", + "daily", "BaseStep", ] diff --git a/reme4/steps/common/__init__.py b/reme4/steps/common/__init__.py index e09c2073..73248fd5 100644 --- a/reme4/steps/common/__init__.py +++ b/reme4/steps/common/__init__.py @@ -1,21 +1,17 @@ """Common steps.""" -from .demo import DemoEchoStep1, DemoEchoStep2 from .health_check import HealthCheckStep from .help import HelpStep from .reindex import ReindexStep from .search import SearchStep -from .stream_demo import StreamDemoStep1, StreamDemoStep2 +from .traverse import TraverseStep from .version import VersionStep __all__ = [ - "DemoEchoStep1", - "DemoEchoStep2", "HealthCheckStep", "HelpStep", "ReindexStep", "SearchStep", - "StreamDemoStep1", - "StreamDemoStep2", + "TraverseStep", "VersionStep", ] diff --git a/reme4/steps/common/health_check.py b/reme4/steps/common/health_check.py index f185b21a..2fefd826 100644 --- a/reme4/steps/common/health_check.py +++ b/reme4/steps/common/health_check.py @@ -5,10 +5,11 @@ from collections.abc import Mapping import numpy as np +from ...enumeration.component_enum import ComponentEnum + from ..base_step import BaseStep from ... import __version__ from ...components import R -from ...enumeration import ComponentEnum def _deep_size(obj, _seen: set | None = None) -> int: diff --git a/reme4/steps/common/reindex.py b/reme4/steps/common/reindex.py index f4d280e9..84f86939 100644 --- a/reme4/steps/common/reindex.py +++ b/reme4/steps/common/reindex.py @@ -1,4 +1,4 @@ -"""Wipe the file store and rebuild it from the watcher's tracked files.""" +"""Wipe the file store and rebuild it by scanning the vault from disk.""" from ..base_step import BaseStep from ...components import R @@ -6,18 +6,29 @@ from ...components import R @R.register("reindex_step") class ReindexStep(BaseStep): - """Full re-index: stop watcher, clear store, sync from disk, then restart.""" + """Full re-index: clear store, walk vault, hand the file list to index_changes.""" async def execute(self): assert self.context is not None - await self.file_watcher.close() - try: - await self.file_store.clear() - counts = await self.file_watcher.update_store() - finally: - await self.file_watcher.start() + suffix_filters: list[str] = self.context.get("suffix_filters", ["md"]) + suffixes = tuple("." + s.strip(".") for s in suffix_filters) if suffix_filters else None + await self.file_store.clear() + + paths: list[str] = [] + for p in self.vault_path.rglob("*"): + if not p.is_file(): + continue + if suffixes and not str(p).endswith(suffixes): + continue + paths.append(str(p.absolute())) + + if paths: + await self.run_job("index_changes", changes=[{"change": "added", "path": p} for p in paths]) + await self.file_store.dump() + + counts = {"added": len(paths), "modified": 0, "deleted": 0} self.logger.info(f"[{self.name}] reindexed {counts}") self.context.response.answer = f"🔄 Reindexed {counts['added']} file(s)" self.context.response.metadata["counts"] = counts diff --git a/reme4/steps/common/search.py b/reme4/steps/common/search.py index da783b64..b942703e 100644 --- a/reme4/steps/common/search.py +++ b/reme4/steps/common/search.py @@ -73,27 +73,25 @@ class SearchStep(BaseStep): @staticmethod def _node_meta(node: FileNode | None) -> dict: - """Extract a compact meta dict (title/description/tags) from a FileNode.""" + """Extract a compact meta dict (name/description) from a FileNode.""" if node is None: return {} fm = node.front_matter meta: dict = {} - if fm.title: - meta["title"] = fm.title + if fm.name: + meta["name"] = fm.name if fm.description: meta["description"] = fm.description - if fm.tags: - meta["tags"] = list(fm.tags) return meta @staticmethod def _format_meta_inline(meta: dict) -> str: """One-line render of node meta for the answer; '(no meta)' when empty.""" parts = [] - if "title" in meta: - parts.append(f'title="{meta["title"]}"') - if "tags" in meta: - parts.append(f"tags={meta['tags']}") + if "name" in meta: + parts.append(f'name="{meta["name"]}"') + if "description" in meta: + parts.append(f'description="{meta["description"]}"') return " ".join(parts) if parts else "(no meta)" @staticmethod @@ -169,7 +167,10 @@ class SearchStep(BaseStep): expand_links: bool = bool(self.kwargs.get("expand_links", True)) max_links_per_direction: int = int(self.kwargs.get("max_links_per_direction", 10)) - assert query, "query cannot be empty" + if not query: + self.context.response.success = False + self.context.response.answer = "Error: query cannot be empty" + return self.context.response assert 0.0 <= vector_weight <= 1.0, f"vector_weight must be in [0, 1], got {vector_weight}" assert limit > 0, f"limit must be positive, got {limit}" diff --git a/reme4/steps/common/traverse.py b/reme4/steps/common/traverse.py new file mode 100644 index 00000000..ae955acf --- /dev/null +++ b/reme4/steps/common/traverse.py @@ -0,0 +1,154 @@ +"""``traverse_step`` — BFS over wikilink edges from a seed file. + +Single tool for relationship browsing. ``depth=1`` covers the trivial +"what does this link to / what links here" lookups (set ``direction`` +accordingly); higher depth opens up multi-hop exploration. + +Output is one record per edge traversed (not per node), so the same +target can appear multiple times if reached via different predicates +or paths — agents dedupe at the call site if they want a flat node +set. Each record carries ``via`` (the predecessor) and the link's +``predicate`` / ``anchor`` so the agent can reconstruct the path. + +Adjacency is loaded once via ``file_graph.get_nodes(None)`` — every +real node arrives with its full ``links`` payload, and we build both +the outbound and the inbound index in a single pass. The BFS then +runs purely in memory: no per-frontier-node graph round-trips, no +filesystem walk. The ``get_inlinks`` / ``get_outlinks`` contract +methods stay unused here because they'd add network round-trips for +data we already have. + +Direction vocabulary accepts both the standard convention +(``forward`` / ``backward`` / ``both``) and the engine convention +(``out`` / ``in`` / ``both``). + +Seeds are paths relative to the vault used as-is — short-form resolution +is no longer attempted. Seeds that don't match any graph node yield +empty BFS results (no error). +""" + +from collections import deque +from pathlib import Path + +from ..base_step import BaseStep + +from ...components import R + +from ...schema import FileLink + + +_FORWARD = {"out", "forward"} +_BACKWARD = {"in", "backward"} +_BOTH = {"both"} +_VALID_DIRECTIONS = _FORWARD | _BACKWARD | _BOTH + + +async def _build_indexes( + file_store, +) -> tuple[ + dict[str, list[tuple[str, FileLink]]], + dict[str, list[tuple[str, FileLink]]], +]: + """One ``get_nodes(None)`` call → (outbound, inbound) adjacency dicts. + + Each dict is keyed by node path; values are ``(neighbor_path, link)`` + tuples. Source paths land in the inbound index alongside the link + object — solving the contract gap where ``get_inlinks`` returns + target-shaped FileLinks without source attribution. + """ + outbound: dict[str, list[tuple[str, FileLink]]] = {} + inbound: dict[str, list[tuple[str, FileLink]]] = {} + if not file_store.file_graph: + return outbound, inbound + for node in await file_store.file_graph.get_nodes(): + for link in node.links: + if not link.target_path: + continue + outbound.setdefault(node.path, []).append((link.target_path, link)) + inbound.setdefault(link.target_path, []).append((node.path, link)) + return outbound, inbound + + +def _bfs( + seeds: list[str], + max_depth: int, + direction: str, + outbound: dict[str, list[tuple[str, FileLink]]], + inbound: dict[str, list[tuple[str, FileLink]]], +) -> list[dict]: + """In-memory BFS. One record per edge traversed.""" + walk_out = direction in _FORWARD or direction in _BOTH + walk_in = direction in _BACKWARD or direction in _BOTH + + visited_edges: set[tuple[str, str, str | None]] = set() + results: list[dict] = [] + queue: deque[tuple[str, int]] = deque((s, 0) for s in seeds) + + while queue: + current, depth = queue.popleft() + if depth >= max_depth: + continue + + edges: list[tuple[str, str | None, str | None]] = [] + if walk_out: + for tgt, link in outbound.get(current, ()): + edges.append((tgt, link.predicate, link.target_anchor)) + if walk_in: + for src, link in inbound.get(current, ()): + edges.append((src, link.predicate, link.target_anchor)) + + for next_path, pred, anchor in edges: + edge_key = (current, next_path, pred) + if edge_key in visited_edges: + continue + visited_edges.add(edge_key) + results.append( + { + "path": next_path, + "depth": depth + 1, + "via": current, + "predicate": pred, + "anchor": anchor, + }, + ) + if depth + 1 < max_depth: + queue.append((next_path, depth + 1)) + + return results + + +def _normalize_seeds(raw) -> list[str]: + """Coerce raw seed input to a non-empty list of strings relative to the vault.""" + if isinstance(raw, (str, Path)): + items = [raw] + else: + items = list(raw or []) + return [str(p) for p in items if p] + + +@R.register("traverse_step") +class TraverseStep(BaseStep): + """BFS from a seed file to explore wikilink relationships. + + Parameters: + path — single seed (str) or a list of seeds. + direction — ``forward`` / ``backward`` / ``both`` (or ``out`` / ``in`` / ``both``). + depth — hop limit (default 1 = immediate neighbors). + """ + + async def execute(self): + assert self.context is not None + seeds_raw = self.context.get("path") + depth = int(self.context.get("depth") or 1) + direction = (self.context.get("direction") or "both").lower() + assert ( + direction in _VALID_DIRECTIONS + ), f"direction must be one of {sorted(_VALID_DIRECTIONS)}, got {direction!r}" + seeds = _normalize_seeds(seeds_raw) + assert seeds, "path is required" + outbound, inbound = await _build_indexes(self.file_store) + results = _bfs(seeds, depth, direction, outbound, inbound) + self.context.response.success = True + seed_label = seeds[0] if len(seeds) == 1 else f"{len(seeds)} seeds" + self.context.response.answer = f"Traversed {len(results)} edge(s) from {seed_label}" + self.context.response.metadata.update({"edges": results, "count": len(results)}) diff --git a/reme4/steps/crud/__init__.py b/reme4/steps/crud/__init__.py index 78414bea..78356a47 100644 --- a/reme4/steps/crud/__init__.py +++ b/reme4/steps/crud/__init__.py @@ -1,13 +1,40 @@ -"""CRUD steps for markdown files under the working_dir.""" +"""File-level ops on vault_dir — both opaque-byte and text-content surfaces. + +The package covers two related surfaces: + +* **Opaque-byte ops** (don't care about file type): ``delete``, + ``download``, ``list``, ``move``, ``stat``, ``upload``, + ``upload_resource``. +* **Text-content ops** (markdown-aware; layered on the same path- + resolution helpers in ``_file_io.py``): ``read``, ``write``, + ``append``, ``edit``. + +For frontmatter slice RUD (YAML structured-data semantics) see +``reme4.steps.frontmatter``. +""" -from .append import AppendStep -from .edit import EditStep from .read import ReadStep +from .edit import EditStep +from .delete import DeleteStep from .write import WriteStep +from .append import AppendStep +from .move import MoveStep +from .stat import StatStep +from .download import DownloadStep +from .list import ListStep +from .upload import UploadStep +from .upload_resource import UploadResourceStep __all__ = [ - "AppendStep", - "EditStep", - "ReadStep", + "DeleteStep", "WriteStep", + "AppendStep", + "MoveStep", + "StatStep", + "DownloadStep", + "ListStep", + "UploadStep", + "UploadResourceStep", + "ReadStep", + "EditStep", ] diff --git a/reme4/steps/crud/delete.py b/reme4/steps/crud/delete.py new file mode 100644 index 00000000..a0b75ea9 --- /dev/null +++ b/reme4/steps/crud/delete.py @@ -0,0 +1,133 @@ +"""``file_delete`` — hard-delete a file or folder under the vault (reports inbound refs). + +Removes the file (or folder tree) from disk; the watcher then prunes +the affected chunks from all projections (vector / keyword / file_graph). +For **soft** deletion that keeps a file addressable, use +``property_update(status=archived)`` instead — that's the path +structure.md's Decay algorithm is designed around. + +Reference reporting (no auto-fix). Before deleting, the inbound +wikilink count for each doomed ``.md`` file is captured from the +file_graph's reverse index (literal full-path matching, same rule +as the wikilink retarget helper). For folder deletes, sources that live inside +the same folder are filtered out — those links die alongside the +delete and can't surface as dangling. The remaining inbound list +is the agent's punch list for follow-up rewrites. + +The delete itself is unconditional: this step does not rewrite +inbound references on the agent's behalf because there is no +canonical "new target" — the agent decides per-reference whether +to rewrite (via ``file_move`` to a merge target), edit the +citing prose, or accept dangling. +""" + +import shutil +from pathlib import Path + +from ..base_step import BaseStep +from ...utils.wikilink_handler import WikilinkHandler + +from ...components import R + + +def _is_inside(rel: str, folder_rel: str) -> bool: + """``rel`` is the same as or nested under ``folder_rel`` (relative to the vault).""" + prefix = folder_rel.rstrip("/") + "/" + return rel == folder_rel or rel.startswith(prefix) + + +@R.register("delete_step") +class DeleteStep(BaseStep): + """Hard-delete the path at ``path`` (file or folder, relative to the vault).""" + + async def execute(self): + assert self.context is not None + path: str = self.context.get("path", "") or "" + assert path, "path is required" + payload = await self._delete(path) + if "error" in payload: + self.context.response.success = False + self.context.response.answer = f"Error: {payload['error']}" + elif payload.get("is_dir"): + self.context.response.success = True + self.context.response.answer = f"Deleted directory {path} ({len(payload['deleted_files'])} file(s))" + else: + self.context.response.success = True + self.context.response.answer = f"Deleted {path}" + self.context.response.metadata.update(payload) + + async def _delete(self, path: str) -> dict: + if not path: + return {"path": path, "error": "not found"} + vault_dir = Path(self.file_store.vault_path or ".").resolve() + target = (vault_dir / path).resolve() + if target.is_file(): + inbound = await WikilinkHandler.find_inbound(self.file_store, target=path) + target.unlink() + return { + "path": path, + "deleted": True, + "is_dir": False, + "deleted_files": [path], + "inbound": { + "files_touched": inbound.get("files_touched", 0), + "links_total": inbound.get("links_total", 0), + "by_file": inbound.get("by_file", []), + }, + } + + if target.is_dir(): + folder_rel = path.rstrip("/") + deleted_files: list[str] = [] + per_target: list[dict] = [] + external_sources: set[str] = set() + links_total = 0 + + for md in sorted(target.rglob("*.md")): + try: + rel = str(md.relative_to(vault_dir)) + except ValueError: + continue + deleted_files.append(rel) + inbound = await WikilinkHandler.find_inbound(self.file_store, target=rel) + # Drop sources that also live inside the doomed folder — + # their links vanish with them and aren't actionable. + external = [row for row in inbound.get("by_file", []) if not _is_inside(row["path"], folder_rel)] + if not external: + continue + target_total = sum(row["count"] for row in external) + links_total += target_total + external_sources.update(row["path"] for row in external) + per_target.append( + { + "target": rel, + "files_touched": len(external), + "links_total": target_total, + "by_file": external, + }, + ) + + # Also enumerate non-md files for the deleted_files report. + for entry in sorted(target.rglob("*")): + if not entry.is_file() or entry.suffix == ".md": + continue + try: + rel = str(entry.relative_to(vault_dir)) + except ValueError: + continue + deleted_files.append(rel) + + shutil.rmtree(target) + return { + "path": path, + "deleted": True, + "is_dir": True, + "deleted_files": sorted(deleted_files), + "inbound": { + "files_touched": len(external_sources), + "links_total": links_total, + "by_target": per_target, + }, + } + + return {"path": path, "error": "not found"} diff --git a/reme4/steps/crud/download.py b/reme4/steps/crud/download.py new file mode 100644 index 00000000..8dca0732 --- /dev/null +++ b/reme4/steps/crud/download.py @@ -0,0 +1,84 @@ +"""``file_download`` — copy a file out of the vault to a local path. + +Symmetric counterpart to ``file_upload``: source is under the vault, +target is on the local filesystem. + +``src_path`` is a path relative to the vault (the file to copy out). +Returns ``error="not found"`` when the file isn't on disk. + +``dst_path`` (filesystem target) is an absolute path on the host +filesystem. **Optional** — when empty, the file lands in a +session-scoped temp dir (lazy, auto-cleaned on process exit; each +call gets its own subdirectory so concurrent agents don't trample +each other) and the realized path is returned in ``dst_path``. +``overwrite`` defaults to False — callers must opt in to clobber an +existing destination. +""" + +import mimetypes +import shutil +import tempfile +from pathlib import Path + +from ..base_step import BaseStep + +from ...components import R + + +_TEMP_ROOT: Path | None = None + + +def _get_temp_root() -> Path: + """Lazy session-scoped temp dir. Auto-cleaned on process exit.""" + global _TEMP_ROOT + if _TEMP_ROOT is None: + _TEMP_ROOT = Path(tempfile.mkdtemp(prefix="reme-files-")) + return _TEMP_ROOT + + +@R.register("download_step") +class DownloadStep(BaseStep): + """Copy ``src_path`` (under the vault) to ``dst_path`` (or a temp file if omitted).""" + + async def execute(self): + assert self.context is not None + src_path: str = self.context.get("src_path", "") or "" + dst_path: str = self.context.get("dst_path", "") or "" + overwrite: bool = bool(self.context.get("overwrite", False)) + assert src_path, "src_path is required" + payload = await self._download(src_path, dst_path, overwrite) + if "error" in payload: + self.context.response.success = False + self.context.response.answer = f"Error: {payload['error']}" + else: + self.context.response.success = True + self.context.response.answer = f"Downloaded {src_path} → {payload['dst_path']} ({payload['size']} bytes)" + self.context.response.metadata.update(payload) + + async def _download(self, src_path: str, dst_path: str, overwrite: bool) -> dict: + if not src_path: + return {"src_path": src_path, "error": "not found"} + src_abs = (Path(self.file_store.vault_path or ".") / src_path).resolve() + if not src_abs.is_file(): + return {"src_path": src_path, "error": "not found"} + + if dst_path: + dst_abs = Path(dst_path) + if dst_abs.exists() and not overwrite: + return { + "src_path": src_path, + "dst_path": dst_path, + "error": "destination exists; pass overwrite=True", + } + dst_abs.parent.mkdir(parents=True, exist_ok=True) + else: + dst_dir = Path(tempfile.mkdtemp(prefix="dl-", dir=_get_temp_root())) + dst_abs = dst_dir / src_abs.name + + shutil.copy2(src_abs, dst_abs) + return { + "src_path": src_path, + "dst_path": str(dst_abs), + "size": dst_abs.stat().st_size, + "mime": mimetypes.guess_type(dst_abs.name)[0] or "application/octet-stream", + } diff --git a/reme4/steps/crud/list.py b/reme4/steps/crud/list.py new file mode 100644 index 00000000..9b5d1b0c --- /dev/null +++ b/reme4/steps/crud/list.py @@ -0,0 +1,55 @@ +"""``file_list`` — enumerate files under a directory in the vault. + +Reads directly from the filesystem (``Path.iterdir`` / +``Path.rglob``), **not** the file_store index. The store may lag +behind disk during indexing or after rapid mutations; for the most +current view, the on-disk walk is the source of truth. + +Parameters: + path — directory to list under (relative to the vault or absolute). + Empty = vault root. + limit — cap the number of returned items. + recursive — descend into subdirectories. Default False = direct + children only. + +No frontmatter is read — this is a plain directory walker. Callers +that need frontmatter-based filtering should iterate the result and +call ``frontmatter_read`` per candidate. +""" + +from pathlib import Path + +from ..base_step import BaseStep + +from ...components import R + + +@R.register("list_step") +class ListStep(BaseStep): + """Enumerate files under a directory in the vault.""" + + async def execute(self): + assert self.context is not None + path: str = self.context.get("path") or "" + recursive: bool = bool(self.context.get("recursive", False)) + limit: int = int(self.context.get("limit") or 100) + + vault_dir = Path(self.file_store.vault_path or ".").resolve() + target_dir = (vault_dir / (path or ".")).resolve() + items: list[str] = [] + if target_dir.is_dir(): + entries = target_dir.rglob("*") if recursive else target_dir.iterdir() + for entry in entries: + if not entry.is_file(): + continue + try: + rel = str(entry.relative_to(vault_dir)) + except ValueError: + rel = str(entry) + items.append(rel) + if len(items) >= limit: + break + + self.context.response.success = True + self.context.response.answer = f"Listed {len(items)} file(s) under {path or '.'}" + self.context.response.metadata.update({"items": items, "count": len(items)}) diff --git a/reme4/steps/crud/move.py b/reme4/steps/crud/move.py new file mode 100644 index 00000000..aaf514de --- /dev/null +++ b/reme4/steps/crud/move.py @@ -0,0 +1,132 @@ +"""``file_move`` — relocate / rename a file within the vault by copy → retarget → delete. + +Three-step ordering keeps vault_dir referentially consistent at every +intermediate point — no window in which inbound ``[[src_path]]`` wikilinks +dangle: + + 1. ``shutil.copyfile(src_path, dst_path)``. Both files now exist on disk; + inbound ``[[src_path]]`` still resolves (to the original location). + 2. ``retarget_links(src_path, dst_path)``. Rewrites every inbound + ``[[src_path]]`` → ``[[dst_path]]`` across the vault. Both files + exist throughout, so rewrites can land in any order without + breaking resolution. + 3. ``src_abs.unlink()``. References now point at ``dst_path``; the + original is an orphan and is safely removed. + +If retargeting fails (raises or returns an error payload), step 3 is +skipped — both files remain on disk so the caller can diagnose and +retry; vault_dir stays consistent (references still resolve to the +original). The ``src_removed`` boolean in the payload distinguishes +the two cases. + +``src_path`` must resolve inside vault_dir as a path relative to the +vault; ``dst_path`` must be relative to the vault with a directory +component (same rule as ``file_upload``). For cross-realm transfer +(vault_dir ↔ local fs) use ``file_upload`` / ``file_download``. + +Opt out via ``retarget=False`` for the rare case where you intentionally +want to leave references stale (e.g. moving aside before delete) — the +original is still removed in that case (move semantics, not copy). +""" + +import shutil +from pathlib import Path + +from ..base_step import BaseStep +from ...utils.wikilink_handler import WikilinkHandler + +from ...components import R + + +@R.register("move_step") +class MoveStep(BaseStep): + """Move ``src_path`` to ``dst_path`` within the vault (copy → retarget → unlink).""" + + async def execute(self): + assert self.context is not None + src_path: str = self.context.get("src_path", "") or "" + dst_path: str = self.context.get("dst_path", "") or "" + overwrite: bool = bool(self.context.get("overwrite", False)) + retarget: bool = bool(self.context.get("retarget", True)) + assert src_path and dst_path, "src_path and dst_path are required" + payload = await self._move(src_path, dst_path, overwrite, retarget) + if "error" in payload: + self.context.response.success = False + self.context.response.answer = f"Error: {payload['error']}" + else: + self.context.response.success = True + self.context.response.answer = f"Moved {src_path} → {dst_path}" + self.context.response.metadata.update(payload) + + async def _move(self, src_path: str, dst_path: str, overwrite: bool, retarget: bool) -> dict: + vault_dir = Path(self.file_store.vault_path or ".") + src_abs = (vault_dir / src_path).resolve() if src_path else None + dst_abs = (vault_dir / dst_path).resolve() if not Path(dst_path).is_absolute() else None + precheck_error = _precheck_move(src_path, dst_path, src_abs, dst_abs, overwrite) + if precheck_error: + return precheck_error + assert src_abs is not None and dst_abs is not None # narrowed by precheck + dst_abs.parent.mkdir(parents=True, exist_ok=True) + + # Step 1 — copy. Both files exist; inbound [[src_path]] still resolves. + shutil.copyfile(str(src_abs), str(dst_abs)) + payload: dict = {"src_path": src_path, "dst_path": dst_path, "size": dst_abs.stat().st_size} + + # Step 2 — retarget. vault_dir stays consistent throughout: refs still + # at [[src_path]] resolve to the original; refs already rewritten to + # [[dst_path]] resolve to the new location. On error, bail before + # unlinking so the caller can retry; vault_dir is still consistent. + if retarget: + try: + report = await WikilinkHandler.retarget_links(self.file_store, src=src_path, dst=dst_path) + except Exception as exc: + payload["retarget"] = {"error": f"retarget raised: {exc!r}"} + payload["src_removed"] = False + return payload + if "error" in report: + payload["retarget"] = report + payload["src_removed"] = False + return payload + payload["retarget"] = { + "files_touched": report.get("files_touched", 0), + "links_changed": report.get("links_changed", 0), + "by_file": report.get("by_file", []), + "ambiguous": report.get("ambiguous", []), + } + else: + payload["retarget"] = None + + # Step 3 — unlink the original. Refs all point at dst_path now; the + # original is an orphan. If unlink fails, vault_dir is still + # consistent (refs resolve to dst_path); the original just lingers + # as an orphan that the caller can clean up. + try: + src_abs.unlink() + payload["src_removed"] = True + except Exception as exc: + payload["src_removed"] = False + payload["src_remove_error"] = f"unlink raised: {exc!r}" + + return payload + + +def _precheck_move( + src_path: str, + dst_path: str, + src_abs: Path | None, + dst_abs: Path | None, + overwrite: bool, +) -> dict | None: + """Validate inputs for ``_move``; return an error payload or ``None`` when OK.""" + if src_abs is None or not src_abs.is_file(): + return {"src_path": src_path, "error": "not found"} + if dst_abs is None or "/" not in dst_path: + return { + "dst_path": dst_path, + "error": "dst_path must be relative to the vault with a directory component", + } + if dst_abs == src_abs: + return {"src_path": src_path, "dst_path": dst_path, "error": "src_path and dst_path are the same"} + if dst_abs.exists() and not overwrite: + return {"dst_path": dst_path, "error": "destination exists; pass overwrite=True"} + return None diff --git a/reme4/steps/crud/stat.py b/reme4/steps/crud/stat.py new file mode 100644 index 00000000..e6518724 --- /dev/null +++ b/reme4/steps/crud/stat.py @@ -0,0 +1,72 @@ +"""``file_stat`` — peek at file metadata under the vault without copying it. + +Cheap inspection alternative to ``file_download``: the agent gets +size, mtime, mime type, and (for markdown files) the parsed +frontmatter — enough to decide whether to download / parse / skip +without paying the copy cost. + +Returns a uniform envelope: + + {exists, type, size, mtime, ctime, mime, frontmatter} + +``exists=False`` short-circuits everything else to ``None``. ``type`` +is ``"file"`` / ``"dir"`` (covers event workspace probes too). +``frontmatter`` is populated only for ``.md`` files and only when +parsing succeeds — schema validity is a lint concern. + +``path`` accepts a file path or directory path relative to the vault. +Joined with ``file_store.vault_path`` and inspected on disk. +""" + +import mimetypes +from datetime import datetime +from pathlib import Path + +import frontmatter + +from ..base_step import BaseStep + +from ...components import R + + +@R.register("stat_step") +class StatStep(BaseStep): + """Return metadata for a file or directory under the vault.""" + + async def execute(self): + assert self.context is not None + path: str = self.context.get("path", "") or "" + assert path, "path is required" + + target = (Path(self.file_store.vault_path or ".") / path).resolve() + if not target.exists(): + self.context.response.success = False + self.context.response.answer = f"stat: {path} not found" + self.context.response.metadata.update({"path": path, "exists": False}) + return + + st = target.stat() + payload: dict = { + "path": path, + "absolute_path": str(target), + "exists": True, + "type": "dir" if target.is_dir() else "file", + "mtime": datetime.fromtimestamp(st.st_mtime).isoformat(), + "ctime": datetime.fromtimestamp(st.st_ctime).isoformat(), + } + if target.is_file(): + payload["size"] = st.st_size + payload["mime"] = mimetypes.guess_type(target.name)[0] or "application/octet-stream" + if target.suffix == ".md": + try: + meta = dict(frontmatter.loads(target.read_text(encoding="utf-8")).metadata) + except Exception: + meta = {} + payload["frontmatter"] = meta + answer = f"stat: {path} (file, {st.st_size} bytes)" + else: + answer = f"stat: {path} (dir)" + + self.context.response.success = True + self.context.response.answer = answer + self.context.response.metadata.update(payload) diff --git a/reme4/steps/crud/upload.py b/reme4/steps/crud/upload.py new file mode 100644 index 00000000..b5a8c0c9 --- /dev/null +++ b/reme4/steps/crud/upload.py @@ -0,0 +1,74 @@ +"""``file_upload`` — copy a file from the local filesystem into the vault. + +Symmetric counterpart to ``file_download``: source is on the local +filesystem, target is under the vault. + +``src_path`` (filesystem source) is an absolute path on the host +filesystem (the file to copy in). Returns ``error="not found"`` +when the file isn't on disk. + +``dst_path`` is a path relative to the vault, **required**, and must +include a directory component so the caller is always explicit about +where in the vault the file lands. ``overwrite`` defaults to False — +callers must opt in to clobber an existing destination. + +For the resource-bucket ingest path (channel-tagged, dated under +``resource//`` with provenance metadata) use +``upload_resource`` instead. +""" + +import mimetypes +import shutil +from pathlib import Path + +from ..base_step import BaseStep + +from ...components import R + + +@R.register("upload_step") +class UploadStep(BaseStep): + """Copy ``src_path`` (on local fs) to ``dst_path`` (under the vault).""" + + async def execute(self): + assert self.context is not None + src_path: str = self.context.get("src_path", "") or "" + dst_path: str = self.context.get("dst_path", "") or "" + overwrite: bool = bool(self.context.get("overwrite", False)) + assert src_path and dst_path, "src_path and dst_path are required" + payload = await self._upload(src_path, dst_path, overwrite) + if "error" in payload: + self.context.response.success = False + self.context.response.answer = f"Error: {payload['error']}" + else: + self.context.response.success = True + self.context.response.answer = f"Uploaded {src_path} → {dst_path} ({payload['size']} bytes)" + self.context.response.metadata.update(payload) + + async def _upload(self, src_path: str, dst_path: str, overwrite: bool) -> dict: + src_abs = Path(src_path) + if not src_abs.is_file(): + return {"src_path": src_path, "error": "not found"} + if "/" not in dst_path: + return { + "dst_path": dst_path, + "error": "dst_path must be relative to the vault with a directory component", + } + vault_dir = Path(self.file_store.vault_path or ".") + dst_abs = (vault_dir / dst_path).resolve() if not Path(dst_path).is_absolute() else None + if dst_abs is None: + return {"dst_path": dst_path, "error": "dst_path must be relative to the vault"} + if dst_abs.exists() and not overwrite: + return { + "src_path": src_path, + "dst_path": dst_path, + "error": "destination exists; pass overwrite=True", + } + dst_abs.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src_abs, dst_abs) + return { + "src_path": src_path, + "dst_path": dst_path, + "size": dst_abs.stat().st_size, + "mime": mimetypes.guess_type(dst_abs.name)[0] or "application/octet-stream", + } diff --git a/reme4/steps/crud/upload_resource.py b/reme4/steps/crud/upload_resource.py new file mode 100644 index 00000000..f732727e --- /dev/null +++ b/reme4/steps/crud/upload_resource.py @@ -0,0 +1,447 @@ +"""``upload_resource`` — copy an externally-received asset into ``resource//``. + +This step is the **passive** ingest entry point: when an external +channel (wechat group, email, browser save, API push, ...) hands +the agent a file, ``upload_resource`` lands it in ``resource//`` +keyed by the day it was received (always today, local time), +alongside a ``meta.json`` row recording provenance. + +This is the only ingest path through ``resource/``. Materials the +main agent *actively* fetches or generates during a daily task +belong inside the daily folder as sibling materials, not here. + +For generic file copy (local fs → arbitrary vault path) use +``upload`` instead. + +Bucket layout:: + + resource// + meta.json # JSON array of FileNode rows + .md # derived markdown view + ____ # the asset itself + ... + +Naming convention — the file name is **derived**, never caller-supplied:: + + ____ + +* ```` — top-level identity (anchors provenance), lowercase + letters/digits/dashes only. +* ```` — receive time within the bucket day (the date is + already implicit in the bucket folder). +* ```` — the basename of the ``path`` argument, + after rejecting path separators, dot segments, and leading dots. + +This format is self-describing in directory listings + wikilinks +(``[[resource//wechat__153022__report.pdf]]``) and makes +cross-channel basename collisions structurally impossible. Two +genuine duplicates (same channel + same second + same basename) +are reported as an error — the step never silently dedupes, so +callers see the conflict and can decide whether to retry, rename +upstream, or skip. + +Each ``upload_resource`` call: + +1. Resolves the bucket date as today (local time). +2. Validates the inputs: ``path`` exists, ``channel`` matches the + allowed character class, ``description`` is non-empty, the + source basename has no path separators / dot segments / leading + dot. +3. Builds the final name from the format above. +4. Under a per-day file lock, checks the final name against + ``meta.json`` ∪ the on-disk listing. Any collision → error + (no silent suffixing). +5. Copies the asset into the bucket. +6. Appends a :class:`FileNode` row to ``meta.json`` (with provenance + on ``front_matter``) and re-renders ``.md``. + +Parameters: + +* ``path`` (required) — local filesystem path to the asset to ingest. +* ``channel`` (required) — inbound channel identifier (wechat / + email / browser / api / ...). Lowercase letters / digits / dashes + only. +* ``description`` (required) — analysis hint for downstream agents: + where the asset came from, what kind of content it carries, and how + it should be interpreted. The digester / synchronizer reads this + verbatim from ``meta.json`` to decide how to read the asset (skim + vs. deep parse, structured extraction vs. summarization, etc.), so + callers should write enough detail to drive that decision — not + just a title. Multi-line is fine; the ``.md`` bullet view + flattens for display while ``meta.json`` preserves the original. +* ``metadata`` (optional dict) — extras persisted on the meta row. + ``source`` (free-form origin within the channel) is conventional; + any other keys pass through verbatim. Keys ``name``, ``channel``, + ``received_at``, ``description`` are reserved. + +Returns ``{date, name, path}`` on success or ``{error}`` on failure. +""" + +import datetime +import fcntl +import json +import os +import re +import shutil +import tempfile +from pathlib import Path + +from ..base_step import BaseStep + +from ...components import R + +from ...schema import FileFrontMatter, FileNode + + +# Channel identifier character class — keeps the derived filename predictable +# and parseable (the `__` separator is also disjoint from this set). +_CHANNEL_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") + +# Keys the step manages itself — callers cannot smuggle them in via metadata. +_RESERVED_METADATA_KEYS = frozenset({"name", "channel", "received_at", "description"}) + + +@R.register("upload_resource_step") +class UploadResourceStep(BaseStep): + """Land an external asset in ``resource//`` and update the day's meta + index.""" + + async def execute(self): + assert self.context is not None + path: str = (self.context.get("path", "") or "").strip() + channel: str = (self.context.get("channel", "") or "").strip() + description: str = (self.context.get("description", "") or "").strip() + metadata_raw = self.context.get("metadata") or {} + + prepared, prep_error = _prepare_inputs(path, channel, description, metadata_raw) + if prep_error: + self._fail({"error": prep_error}) + return + + try: + outcome = self._land( + src=prepared["src"], + date=prepared["date"], + final_name=prepared["final_name"], + entry_fields={ + **prepared["metadata"], + "channel": channel, + "received_at": prepared["received_at"], + "description": description, + }, + ) + except _DuplicateUpload as e: + self._fail({"error": str(e)}) + return + except Exception as e: + self._fail({"error": f"{type(e).__name__}: {e}"}) + return + + self.context.response.success = True + self.context.response.answer = f"Uploaded {outcome['name']} to {outcome['path']}" + self.context.response.metadata.update(outcome) + + # ------------------------------------------------------------------ + + def _fail(self, payload: dict) -> None: + assert self.context is not None + self.context.response.success = False + self.context.response.answer = f"Error: {payload.get('error', 'upload failed')}" + self.context.response.metadata.update(payload) + + def _resource_dir_name(self) -> str: + """Configured ``resource_dir`` subdir name; defaults to ``"resource"`` outside an app context.""" + if self.app_context is None: + return "resource" + return self.app_context.app_config.resource_dir + + def _vault_dir(self) -> Path: + vr = getattr(self.file_store, "vault_path", None) + return Path(vr).resolve() if vr else Path.cwd().resolve() + + def _land(self, src: Path, date: str, final_name: str, entry_fields: dict) -> dict: + resource_dir = self._resource_dir_name() + bucket = self._vault_dir() / resource_dir / date + bucket.mkdir(parents=True, exist_ok=True) + meta_path = bucket / "meta.json" + day_md = bucket / f"{date}.md" + lock_path = bucket / ".lock" + + rel_path = f"{resource_dir}/{date}/{final_name}" + + with _bucket_lock(lock_path): + existing_entries = _read_meta(meta_path) + # Collision check spans meta ∪ on-disk listing so a stray file + # (from a crashed earlier run) and case-insensitive filesystems + # both surface the conflict rather than getting silently clobbered. + on_disk = {p.name for p in bucket.iterdir() if p.is_file()} + existing_names = {Path(e.path).name for e in existing_entries} | on_disk + if final_name in existing_names: + raise _DuplicateUpload( + f"duplicate: {final_name!r} already exists in {resource_dir}/{date}/", + ) + + dst_path = bucket / final_name + shutil.copyfile(src, dst_path) + + # FileFrontMatter has first-class `name` / `description`; everything + # else (channel / source / received_at / passthrough metadata) rides + # the extras bag (model_config extra="allow"). + description = entry_fields.pop("description", "") + entry = FileNode( + path=rel_path, + st_mtime=dst_path.stat().st_mtime, + front_matter=FileFrontMatter(description=description, **entry_fields), + ) + updated = existing_entries + [entry] + _atomic_write_text( + meta_path, + json.dumps([e.model_dump() for e in updated], ensure_ascii=False, indent=2) + "\n", + ) + _atomic_write_text(day_md, _assemble_day_md(updated, date)) + + return { + "date": date, + "name": final_name, + "path": rel_path, + } + + +class _DuplicateUpload(Exception): + """Raised when the derived name already exists in the bucket.""" + + +# ---------------------------------------------------------------------- +# Input validation +# ---------------------------------------------------------------------- + + +def _prepare_inputs( + path: str, + channel: str, + description: str, + metadata_raw, +) -> tuple[dict, str]: + """Validate caller args and derive the bucket date / final name. + + Returns ``(prepared, error)``: on success ``prepared`` has + ``{src, date, received_at, final_name, metadata}`` and ``error`` is + empty; on failure ``prepared`` is ``{}`` and ``error`` carries the + first violation in user-facing order (path → channel → description + → metadata-shape → file existence → basename → metadata-keys). + """ + src = Path(path) if path else None + metadata, meta_error = _sanitize_metadata(metadata_raw) if isinstance(metadata_raw, dict) else ({}, "") + error = next( + ( + msg + for msg in ( + "path is required" if not path else "", + _validate_channel(channel), + "description is required" if not description else "", + "metadata must be a dict" if not isinstance(metadata_raw, dict) else "", + f"path not found: {path}" if src is not None and not src.is_file() else "", + _validate_basename(src.name) if src is not None else "", + meta_error, + ) + if msg + ), + "", + ) + if error: + return {}, error + + assert src is not None # narrowed by the "path is required" check + now = datetime.datetime.now() + return ( + { + "src": src, + "date": now.strftime("%Y-%m-%d"), + "received_at": now.isoformat(timespec="seconds"), + "final_name": f"{channel}__{now.strftime('%H%M%S')}__{src.name}", + "metadata": metadata, + }, + "", + ) + + +def _validate_channel(channel: str) -> str: + """Return an error string when ``channel`` is unsafe; empty when OK.""" + if not channel: + return "channel is required" + if not _CHANNEL_RE.match(channel): + return f"channel {channel!r} must be lowercase letters / digits / dashes " f"and start with a letter or digit" + return "" + + +def _validate_basename(name: str) -> str: + """Return an error string when the source basename is unsafe; empty when OK. + + The derived filename embeds this string after a ``__`` separator, so we + only need to block characters that would mangle the filesystem path — + path separators, dot segments, leading dots. Bookkeeping-name collisions + (``meta.json`` / ``.md``) are impossible by construction once the + channel + time prefix is prepended. + """ + if not name: + return "source basename is empty" + if "/" in name or "\\" in name: + return f"source basename must not contain path separators: {name!r}" + if name in {".", ".."}: + return f"source basename {name!r} is reserved" + if name.startswith("."): + return f"source basename may not start with '.': {name!r}" + return "" + + +def _sanitize_metadata(raw: dict) -> tuple[dict, str]: + """Return ``(cleaned_metadata, error)``. + + Rejects reserved keys (those the step manages itself) and coerces + ``source`` to a string. All other keys pass through verbatim so + callers can attach arbitrary tags that land on the entry's + ``front_matter`` extras. + """ + for key in _RESERVED_METADATA_KEYS: + if key in raw: + return {}, f"metadata key {key!r} is reserved" + + cleaned = dict(raw) + if "source" in cleaned: + src = cleaned["source"] + cleaned["source"] = src.strip() if isinstance(src, str) else "" + return cleaned, "" + + +# ---------------------------------------------------------------------- +# Per-day exclusive lock + atomic write helpers +# ---------------------------------------------------------------------- + + +class _bucket_lock: + """Exclusive ``flock`` on a per-day lock file; serializes meta+md writes.""" + + def __init__(self, lock_path: Path): + self.lock_path = lock_path + self._fp = None + + def __enter__(self): + self._fp = open(self.lock_path, "w", encoding="utf-8") + fcntl.flock(self._fp.fileno(), fcntl.LOCK_EX) + return self + + def __exit__(self, *exc): + if self._fp is not None: + try: + fcntl.flock(self._fp.fileno(), fcntl.LOCK_UN) + finally: + self._fp.close() + self._fp = None + + +def _read_meta(meta_path: Path) -> list[FileNode]: + """Read ``meta.json`` as a list of :class:`FileNode` rows; missing or malformed → [].""" + if not meta_path.is_file(): + return [] + try: + raw = json.loads(meta_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return [] + if not isinstance(raw, list): + return [] + out: list[FileNode] = [] + for row in raw: + if not isinstance(row, dict): + continue + try: + out.append(FileNode(**row)) + except Exception: + continue + return out + + +def _atomic_write_text(target: Path, text: str) -> None: + """Atomic text write via tempfile + os.replace in the same directory.""" + target.parent.mkdir(parents=True, exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=".upload-", dir=target.parent) + try: + with os.fdopen(tmp_fd, "w", encoding="utf-8") as f: + f.write(text) + os.replace(tmp_path, target) + except Exception: + try: + os.unlink(tmp_path) + except OSError: + pass + raise + + +# ---------------------------------------------------------------------- +# Day-view markdown rendering (pure) +# ---------------------------------------------------------------------- + + +def _assemble_day_md(entries: list[FileNode], date: str) -> str: + """Render the day's bucket as a markdown view. + + Layout:: + + --- + name: + assets: [, , ...] + --- + + # resources + + - [[]] — from `` at — + + Provenance lives on each entry's ``front_matter`` (``channel`` / + ``source`` / ``received_at`` as extras, ``description`` as the + first-class field). ``received_at`` is rendered as ``HH:MM`` when it + parses as ISO 8601, else dropped silently — the asset list stays + readable even when upstream channels emit malformed timestamps. + ``source`` is dropped when empty. ``description`` is flattened + (newlines collapsed to spaces) so the bullet stays one line per + asset; ``meta.json`` preserves the verbatim multi-line text. + """ + names = [Path(e.path).name for e in entries] + lines: list[str] = [ + "---", + f"name: {date}", + f"assets: [{', '.join(names)}]", + "---", + "", + f"# {date} resources", + "", + ] + for entry in entries: + fm = entry.front_matter + channel = getattr(fm, "channel", "") or "" + source = getattr(fm, "source", "") or "" + received_at = getattr(fm, "received_at", "") or "" + description = fm.description or "" + + bits: list[str] = [f"- [[{entry.path}]]"] + provenance = channel + if source: + provenance += f" from `{source}`" + time_part = _hhmm(received_at) + if time_part: + provenance += f" at {time_part}" + bits.append(provenance) + if description: + # Flatten so the bullet stays one line per asset; meta.json keeps + # the verbatim multi-line description for downstream consumers. + bits.append(" ".join(description.split())) + lines.append(" — ".join(bits)) + return "\n".join(lines) + "\n" + + +def _hhmm(received_at: str) -> str: + """Best-effort HH:MM extraction from an ISO 8601 timestamp.""" + if not received_at: + return "" + raw = received_at.replace("Z", "+00:00") + try: + return datetime.datetime.fromisoformat(raw).strftime("%H:%M") + except ValueError: + return "" diff --git a/reme4/steps/daily/__init__.py b/reme4/steps/daily/__init__.py new file mode 100644 index 00000000..9915c778 --- /dev/null +++ b/reme4/steps/daily/__init__.py @@ -0,0 +1,44 @@ +"""Daily-aware steps — note + day-level index, on top of generic file ops. + +A daily note is the single file ``daily//.md``. +The day-level index ``daily/.md`` aggregates that day's +notes into a richer overview page: note list with name/description. +The index is a derived artifact — its source of truth lives in each +note's frontmatter and outlinks; refreshes are idempotent and preserve +manual annotations in marker-delimited sections. + +Tool boundary. The daily module exposes only the operations whose shape +is note- or day-specific: + +* ``daily_resolve_step`` — note path resolver: ensures the day + folder ``daily//`` exists and returns the vault-relative + path ``daily//.md``. Pure path-shape helper — no + body, frontmatter, or index writes (those go through the generic + CRUD + reindex steps). +* ``daily_create_step`` — write the note stub ``daily//.md`` + with a minimal ``name`` frontmatter and refresh the day index. + Idempotent: existing file is left untouched; the index still + refreshes (cheap self-healing). +* ``daily_list_step`` — list the notes under a single day + (defaults to today); returns ``{date, notes: [{path, name, + description}, ...]}``. Also rebuilds ``daily/.md`` as a side + effect (idempotent — the freshly-rendered inventory is what callers + want). Read view of the same operation ``daily_reindex_step`` exposes + from the write side. +* ``daily_reindex_step`` — explicit, idempotent rebuild of a day's index + (historical backfill, drift recovery, batch-create reindex). Returns + the write-result fields ``{date, path, created, notes_count}``. + +Body reads / writes / appends / overwrites all go through the generic +``file_read`` / ``file_write`` tools. Frontmatter edits go through +``property:update``. The day-index is rebuilt explicitly via +``daily_reindex`` after a batch of mutations. +""" + +# Module name 'list' mirrors its tool name. +# pylint: disable=redefined-builtin + +from . import resolve # noqa: F401 -- @R.register("daily_resolve_step") +from . import create # noqa: F401 -- @R.register("daily_create_step") +from . import list # noqa: F401 -- @R.register("daily_list_step") +from . import reindex # noqa: F401 -- @R.register("daily_reindex_step") diff --git a/reme4/steps/daily/_day_index.py b/reme4/steps/daily/_day_index.py new file mode 100644 index 00000000..132879b5 --- /dev/null +++ b/reme4/steps/daily/_day_index.py @@ -0,0 +1,254 @@ +"""``_day_index`` — internal helper: build/refresh ``daily/.md`` index page. + +The day index is a derived artifact whose single job is **daily-note +consolidation** — its source of truth lives in each note's +frontmatter. This module rebuilds the auto-managed sections of the +index page while preserving any manual content the user has added +between markers. + +Frontmatter shape — only the two reserved fields:: + + name: + description: + +The note inventory lives in the body's ```` +wikilinks (graph edges feed off them). No bespoke status / lifecycle +/ scope / role / source / created axes — those are user-defined and +intentionally absent from the auto-managed payload. + +Body auto sections (rebuilt on every refresh, marker-delimited): + +* ``notes`` — bulleted list of ``[[link]]\\n name — description`` rows + +Manual sections live outside the auto markers and are preserved verbatim +across refreshes. A fresh day file gets a ``## 备忘`` section seeded as +the manual scratch area. + +Entry point: ``refresh_day_index(file_store, date)`` — idempotent, safe +to call after every note mutation. ``daily_reindex_step`` exposes +it as a standalone tool; orchestrators (synchronizer, batch flows) call +it explicitly after they finish writing. +""" + +import re +from pathlib import Path + +import frontmatter + +# Marker syntax: HTML comments so they're invisible in rendered markdown +# but trivially detectable in source. Each block has a paired open/close. +_BLOCK_NAMES = ("notes",) +_BLOCK_OPEN = "" +_BLOCK_CLOSE = "" + +_HEADINGS = { + "notes": "## 今日笔记", +} + +_MANUAL_HEADING = "## 备忘" +_MANUAL_STUB = "（人工记录区，刷新索引时不会动）" + + +def _block_re(name: str) -> re.Pattern: + """Capturing regex for an auto block: heading + open marker + inner + close.""" + return re.compile( + rf"(?P^{re.escape(_HEADINGS[name])}\s*\n)?" + rf"{re.escape(_BLOCK_OPEN.format(name=name))}" + r"(?P.*?)" + rf"{re.escape(_BLOCK_CLOSE.format(name=name))}", + re.DOTALL | re.MULTILINE, + ) + + +def _count_digest(n: int) -> str: + """One-line note count, used as the index ``description``.""" + if n == 0: + return "本日暂无笔记。" + return f"今日 {n} 篇笔记。" + + +def _scan_notes(vault_dir: Path, date: str, daily_dir: str) -> list[dict]: + """Walk ``//*.md`` and pull each note's frontmatter. + + Returns one dict per note:: + + {"slug": str, "path": str, "name": str, "description": str} + + Each ``.md`` directly under the day folder is a note; the file's + stem is the slug. Only reserved fields (name / description) are + read — user-defined frontmatter keys are ignored by the index. + """ + date_dir = vault_dir / daily_dir / date + if not date_dir.is_dir(): + return [] + out: list[dict] = [] + for md_path in sorted(p for p in date_dir.iterdir() if p.is_file() and p.suffix == ".md"): + slug = md_path.stem + try: + post = frontmatter.loads(md_path.read_text(encoding="utf-8")) + except Exception: + continue + meta = post.metadata or {} + out.append( + { + "slug": slug, + "path": f"{daily_dir}/{date}/{slug}.md", + "name": str(meta.get("name") or slug), + "description": str(meta.get("description") or "").strip(), + }, + ) + return out + + +def _render_notes_block(notes: list[dict]) -> str: + """Bulleted note digest: link on the bullet line, then an indented + ``name — description`` summary so an agent can scan "what's + happening today" without opening each note. + + The indented summary is omitted entirely when both name and + description add no information beyond the slug already shown in + the link. + """ + if not notes: + return "（无）" + lines: list[str] = [] + for note in notes: + lines.append(f"- [[{note['path']}]]") + name = note["name"] if note["name"] and note["name"] != note["slug"] else "" + description = note["description"] + if name and description: + lines.append(f" {name} — {description}") + elif name: + lines.append(f" {name}") + elif description: + lines.append(f" {description}") + return "\n".join(lines) + + +def _wrap_block(name: str, inner: str) -> str: + """Wrap rendered inner content with heading + auto markers.""" + return f"{_HEADINGS[name]}\n" f"{_BLOCK_OPEN.format(name=name)}\n" f"{inner}\n" f"{_BLOCK_CLOSE.format(name=name)}" + + +def _replace_or_append(body: str, name: str, fresh_block: str) -> str: + """Replace an existing auto block in-place; append at end if absent. + + The replacement keeps the user's heading line if they renamed the + auto-heading (we only own the marker-wrapped inner). Appending uses + our canonical heading + markers so future refreshes find them. + """ + pattern = _block_re(name) + if pattern.search(body): + replacement = f"{_BLOCK_OPEN.format(name=name)}\n" f"{fresh_block}\n" f"{_BLOCK_CLOSE.format(name=name)}" + # Preserve the heading the user had (if any) by only swapping + # the marker-wrapped portion. + return pattern.sub( + lambda m: (m.group("heading") or "") + replacement, + body, + count=1, + ) + # Not present — append the canonical heading + block at the tail. + suffix = _wrap_block(name, fresh_block) + return f"{body.rstrip()}\n\n{suffix}\n" if body.strip() else f"{suffix}\n" + + +def _seed_body(blocks: dict[str, str]) -> str: + """Fresh-file body: all auto blocks in canonical order + manual stub.""" + parts = [_wrap_block(name, blocks[name]) for name in _BLOCK_NAMES] + parts.append(f"{_MANUAL_HEADING}\n{_MANUAL_STUB}") + return "\n\n".join(parts) + "\n" + + +def _merge_blocks(body: str, blocks: dict[str, str]) -> str: + """Refresh every auto block in-place; never touch manual content.""" + for name in _BLOCK_NAMES: + body = _replace_or_append(body, name, blocks[name]) + return body + + +def _frontmatter_payload(date: str, notes: list[dict]) -> dict: + """Reserved-field-only frontmatter for the index page. + + Emits ``name`` / ``description`` and nothing else — other axes + (status / lifecycle / scope / role / source / created) are + user-defined and belong in note bodies, not in this derived + aggregate. + """ + return { + "name": date, + "description": _count_digest(len(notes)), + } + + +async def refresh_day_index(file_store, date: str, daily_dir: str = "daily") -> dict: + """Rebuild ``/.md`` from the current state of its notes. + + Behaviour: + * No ``//`` at all and no existing index file → no-op. + * Notes present → write the index file (create if missing, + otherwise merge auto blocks into the existing body, preserve + manual segments, refresh frontmatter). + * Notes directory empty but index file exists → rebuild with + empty auto blocks (keeps the file in sync with reality). + + ``daily_dir`` defaults to ``"daily"`` for tests / pure-helper + consumers; the registered steps pass the configured + ``application_config.daily_dir`` so the on-disk layout always + matches what the index file claims. + + Returns:: + + { + "date": str, + "path": "/.md", + "notes": [ + {"path": "//.md", + "name": str, + "description": str}, + ... + ], + "created": bool, # True if index file was just written for the first time + } + + The ``notes`` list mirrors the order rendered in the index body + (sorted by slug). The ``created`` field reflects index-page creation, + not note creation, so callers can log "index emerged" events + distinctly. + """ + vault_dir = Path(file_store.vault_path or ".").resolve() + index_rel = f"{daily_dir}/{date}.md" + index_abs = vault_dir / index_rel + notes = _scan_notes(vault_dir, date, daily_dir) + + notes_payload = [{"path": n["path"], "name": n["name"], "description": n["description"]} for n in notes] + + # Nothing to index and no prior index file — quietly do nothing. + if not notes and not index_abs.is_file(): + return { + "date": date, + "path": index_rel, + "notes": notes_payload, + "created": False, + } + + blocks = {"notes": _render_notes_block(notes)} + + if index_abs.is_file(): + post = frontmatter.loads(index_abs.read_text(encoding="utf-8")) + new_body = _merge_blocks(post.content, blocks) + was_created = False + else: + index_abs.parent.mkdir(parents=True, exist_ok=True) + new_body = _seed_body(blocks) + was_created = True + + fm = _frontmatter_payload(date, notes) + out = frontmatter.Post(new_body, **fm) + index_abs.write_text(frontmatter.dumps(out), encoding="utf-8") + + return { + "date": date, + "path": index_rel, + "notes": notes_payload, + "created": was_created, + } diff --git a/reme4/steps/daily/create.py b/reme4/steps/daily/create.py new file mode 100644 index 00000000..fad9c106 --- /dev/null +++ b/reme4/steps/daily/create.py @@ -0,0 +1,84 @@ +"""``daily_create`` — create a daily note file + refresh the day index. + +A daily note is the single file ``daily//.md``. +The day-level index ``daily/.md`` is also refreshed so +all index views (navigation, search, distill input) reflect the new +note. + +This step bakes the conventions an agent shouldn't have to memorize +on every call: + +- path template (``date + slug → daily//.md``) +- reserved-field default: ``name`` falls back to ``slug`` +- day-index refresh so ``daily/.md`` stays consistent + +Frontmatter is intentionally minimal: only the reserved ``name`` +field is written by default. Anything else (status, lifecycle, scope, +role, created, ...) is user-defined — supply it via the generic +``property:update`` step after creation. + +It does NOT do content R-M-W. Once the note exists, the agent +uses ``file_write`` for body edits and ``property:update`` for +frontmatter tweaks. + +Idempotent: when the note file already exists, returns +``{created: False, ...}`` without touching it. The day index is still +refreshed because sibling notes may have changed since the last +call — keeping the index in sync is cheap and self-healing. Pass +``refresh_index=False`` to skip the refresh (rare; mostly for tests / +batch-create flows where the caller will refresh once at the end). +""" + +from datetime import date as _date +from pathlib import Path + +import frontmatter + +from ._day_index import refresh_day_index +from ..base_step import BaseStep + +from ...components import R + + +@R.register("daily_create_step") +class DailyCreateStep(BaseStep): + """Create the note file ``daily//.md`` (idempotent); refresh day index.""" + + async def execute(self): + assert self.context is not None + slug: str = self.context.get("slug", "") or "" + body: str = self.context.get("body", "") or "" + day: str = self.context.get("date") or _date.today().isoformat() + name: str = self.context.get("name", "") or "" + refresh_index: bool = bool(self.context.get("refresh_index", True)) + assert slug, "slug is required" + + daily_dir = self.app_context.app_config.daily_dir if self.app_context is not None else "daily" + path_rel = f"{daily_dir}/{day}/{slug}.md" + vault_dir = Path(self.file_store.vault_path or ".") + path_abs = (vault_dir / path_rel).resolve() + + # Idempotent: existing note returns "already exists" — caller + # decides whether to edit (file_read + file_write) or skip. + already_existed = path_abs.is_file() + if not already_existed: + path_abs.parent.mkdir(parents=True, exist_ok=True) + post = frontmatter.Post(body, name=name or slug) + path_abs.write_text(frontmatter.dumps(post), encoding="utf-8") + + payload: dict = { + "date": day, + "slug": slug, + "path": path_rel, + "created": not already_existed, + } + + # Refresh even on the idempotent path: sibling notes may have + # changed since the last call and the index should track. + if refresh_index: + payload["index"] = await refresh_day_index(self.file_store, day, daily_dir) + + self.context.response.success = True + verb = "Created" if not already_existed else "Reused existing" + self.context.response.answer = f"{verb} daily note {path_rel}" + self.context.response.metadata.update(payload) diff --git a/reme4/steps/daily/list.py b/reme4/steps/daily/list.py new file mode 100644 index 00000000..aec76474 --- /dev/null +++ b/reme4/steps/daily/list.py @@ -0,0 +1,45 @@ +"""``daily_list`` — list the notes under a single day. + +Always rebuilds the day index ``daily/.md`` as a side effect (the +freshly-rendered note inventory is exactly what the caller is asking +to see), then returns one row per ``daily//.md`` note +file with its vault-relative ``path`` plus ``name`` / ``description`` +from frontmatter. + +Distinct from :mod:`daily_reindex` even though both call +``refresh_day_index``: this one is the read view (consumers want the +note inventory), so the index-page bookkeeping fields (``path`` of +``daily/.md``, ``created``) are stripped from the response; +``daily_reindex`` is the write view (consumers want to know what was +rebuilt) and returns those fields without the per-note list. + +Input is a single optional ``date`` (ISO ``YYYY-MM-DD``); falls back to +today. +""" + +from datetime import date as _date + +from ._day_index import refresh_day_index +from ..base_step import BaseStep + +from ...components import R + + +@R.register("daily_list_step") +class DailyListStep(BaseStep): + """List the notes under a single day; also refreshes ``daily/.md``.""" + + async def execute(self): + assert self.context is not None + day: str = (self.context.get("date") or "").strip() or _date.today().isoformat() + daily_dir = self.app_context.app_config.daily_dir if self.app_context is not None else "daily" + refreshed = await refresh_day_index(self.file_store, day, daily_dir) + if "error" in refreshed: + self.context.response.success = False + self.context.response.answer = f"Error: {refreshed['error']}" + self.context.response.metadata.update(refreshed) + return + notes = refreshed["notes"] + self.context.response.success = True + self.context.response.answer = f"Listed {len(notes)} note(s) for {refreshed['date']}" + self.context.response.metadata.update({"date": refreshed["date"], "notes": notes}) diff --git a/reme4/steps/daily/reindex.py b/reme4/steps/daily/reindex.py new file mode 100644 index 00000000..6ff3ed40 --- /dev/null +++ b/reme4/steps/daily/reindex.py @@ -0,0 +1,55 @@ +"""``daily_reindex_step`` — rebuild ``daily/.md`` from its notes. + +The day index ``daily/.md`` is a derived artifact whose job is to +list and describe every note file under ``daily//``. It is **not** +auto-refreshed — ``daily_resolve`` (path resolver), ``file_write`` and +``frontmatter_update`` all leave it stale. This step is the standalone +writer that rebuilds it for batch flows (historical backfill, drift +recovery, end-of-batch consolidation). + +The same rebuild also runs as a side effect of :mod:`daily_list`. The +two steps differ in their response: this one is the **write view** — +it reports the index-page path and a ``created`` flag (true when the +file was just emitted for the first time), which is what a caller +running a rebuild wants to confirm. ``daily_list`` is the **read view** +and returns the per-note inventory instead. + +Input is a single optional ``date`` (ISO ``YYYY-MM-DD``); falls back to +today. + +Always idempotent and safe to re-run. +""" + +from datetime import date as _date + +from ._day_index import refresh_day_index +from ..base_step import BaseStep + +from ...components import R + + +@R.register("daily_reindex_step") +class DailyReindexStep(BaseStep): + """Rebuild ``daily/.md`` from the current state of its notes.""" + + async def execute(self): + assert self.context is not None + day: str = (self.context.get("date") or "").strip() or _date.today().isoformat() + daily_dir = self.app_context.app_config.daily_dir if self.app_context is not None else "daily" + refreshed = await refresh_day_index(self.file_store, day, daily_dir) + if "error" in refreshed: + self.context.response.success = False + self.context.response.answer = f"Error: {refreshed['error']}" + self.context.response.metadata.update(refreshed) + return + notes_count = len(refreshed["notes"]) + self.context.response.success = True + self.context.response.answer = f"Reindexed {refreshed['path']} ({notes_count} note(s))" + self.context.response.metadata.update( + { + "date": refreshed["date"], + "path": refreshed["path"], + "created": refreshed["created"], + "notes_count": notes_count, + }, + ) diff --git a/reme4/steps/daily/resolve.py b/reme4/steps/daily/resolve.py new file mode 100644 index 00000000..6a1523a6 --- /dev/null +++ b/reme4/steps/daily/resolve.py @@ -0,0 +1,91 @@ +"""``daily_resolve`` — resolve a daily note path; ensure the parent day folder exists. + +A daily note is a single markdown file ``daily//.md``. +This step validates ``name``, makes sure the day folder ``daily//`` +exists (so a subsequent ``file_write`` succeeds), and returns the +vault-relative path to the note file. + +Input is a single ``name`` (the note slug). It must be safe to use as a +filename on all platforms — Windows is the strictest, so we validate +against its rules: + +- no reserved characters: ``< > : " / \\ | ? *`` or control chars (``\\x00-\\x1f``) +- no reserved device names: ``CON``, ``PRN``, ``AUX``, ``NUL``, ``COM1-9``, ``LPT1-9`` +- no trailing ``.`` or whitespace +- no leading/trailing whitespace +- non-empty + +Idempotent: returns ``{exists: True}`` when the note file already +exists (caller should read-modify rather than overwrite); otherwise +``{exists: False}`` — the file itself is **not** created here, use +``daily_create`` or ``file_write`` for that. +""" + +import re +from datetime import date as _date +from pathlib import Path + +from ..base_step import BaseStep + +from ...components import R + + +_INVALID_CHARS = re.compile(r'[<>:"/\\|?*\x00-\x1f]') +_RESERVED_NAMES = { + "CON", + "PRN", + "AUX", + "NUL", + *(f"COM{i}" for i in range(1, 10)), + *(f"LPT{i}" for i in range(1, 10)), +} + + +@R.register("daily_resolve_step") +class DailyResolveStep(BaseStep): + """Ensure ``daily//`` exists; return the vault-relative path to ``.md``.""" + + async def execute(self): + assert self.context is not None + name: str = self.context.get("name", "") or "" + + err: str | None = None + if not name: + err = "name is required" + elif name != name.strip(): + err = f"name cannot have leading or trailing whitespace: {name!r}" + elif _INVALID_CHARS.search(name): + err = f'name contains invalid characters (one of < > : " / \\ | ? * or a control char): {name!r}' + elif name.endswith("."): + err = f"name cannot end with '.': {name!r}" + # Windows reserves these device names with or without an extension (CON.txt also forbidden). + elif name.split(".", 1)[0].upper() in _RESERVED_NAMES: + err = f"name is a Windows-reserved device name: {name!r}" + + if err: + self.context.response.success = False + self.context.response.answer = f"Error: {err}" + self.context.response.metadata.update({"error": err}) + return + + day = _date.today().isoformat() + daily_dir = self.app_context.app_config.daily_dir if self.app_context is not None else "daily" + path_rel = f"{daily_dir}/{day}/{name}.md" + vault_dir = Path(self.file_store.vault_path or ".") + path_abs = (vault_dir / path_rel).resolve() + path_abs.parent.mkdir(parents=True, exist_ok=True) + + exists = path_abs.is_file() + payload: dict = { + "date": day, + "name": name, + "path": path_rel, + "exists": exists, + } + if exists: + payload["message"] = f"note already exists at {path_rel}" + + self.context.response.success = True + verb = "Resolved" if not exists else "Resolved existing" + self.context.response.answer = f"{verb} note {path_rel}" + self.context.response.metadata.update(payload) diff --git a/reme4/steps/frontmatter/__init__.py b/reme4/steps/frontmatter/__init__.py new file mode 100644 index 00000000..9ffbed4f --- /dev/null +++ b/reme4/steps/frontmatter/__init__.py @@ -0,0 +1,24 @@ +"""Frontmatter steps — RUD on the YAML frontmatter slice of a markdown file. + +Three Steps: + + frontmatter_read_step — return the frontmatter dict + frontmatter_update_step — merge a patch into the frontmatter + frontmatter_delete_step — drop the listed keys + +Operates only on ``.md`` files; non-markdown targets get +``error="not markdown"`` and the call is **not** executed. + +Body content stays untouched; the sibling ``file`` package owns every +other file-level surface — opaque-byte ops (list / stat / move / +delete / upload / download) and whole-file text ops (read / write / +append / edit). For mid-body edits, use ``file.edit`` (exact string +replacement) or do a read + write round-trip. + +Each Step here is a pure disk read-modify-write — the watcher / parser +notices the change and refreshes the projections asynchronously. +""" + +from . import read # noqa: F401 -- @R.register("frontmatter_read_step") +from . import update # noqa: F401 -- @R.register("frontmatter_update_step") +from . import delete # noqa: F401 -- @R.register("frontmatter_delete_step") diff --git a/reme4/steps/frontmatter/delete.py b/reme4/steps/frontmatter/delete.py new file mode 100644 index 00000000..65367ce1 --- /dev/null +++ b/reme4/steps/frontmatter/delete.py @@ -0,0 +1,66 @@ +"""``frontmatter_delete_step`` — remove keys from a markdown file's frontmatter. + +Returns both ``deleted`` (keys that were present and removed) and +``missing`` (keys that weren't there) so the agent can tell whether +a no-op happened. The file is rewritten only when at least one key +is actually removed — calling delete with all-missing keys is a +zero-side-effect read. + +``path`` is a path relative to the vault. +""" + +from pathlib import Path + +import frontmatter + +from ..base_step import BaseStep + +from ...components import R + + +@R.register("frontmatter_delete_step") +class FrontmatterDeleteStep(BaseStep): + """Remove keys from a markdown file's frontmatter.""" + + async def execute(self): + assert self.context is not None + path: str = self.context.get("path") or "" + assert path, "path is required" + keys = self.context.get("keys") or [] + if isinstance(keys, str): + keys = [keys] + keys = list(keys) + + target = (Path(self.file_store.vault_path or ".") / path).resolve() + if not target.is_file(): + payload: dict = {"path": path, "error": "not found"} + elif target.suffix != ".md": + payload = {"path": path, "error": "not markdown"} + elif not keys: + payload = {"path": path, "error": "keys is empty"} + else: + post = frontmatter.loads(target.read_text(encoding="utf-8")) + deleted: list[str] = [] + missing: list[str] = [] + for k in keys: + if k in post.metadata: + del post.metadata[k] + deleted.append(k) + else: + missing.append(k) + if deleted: + target.write_text(frontmatter.dumps(post), encoding="utf-8") + payload = { + "path": path, + "deleted": deleted, + "missing": missing, + "frontmatter": dict(post.metadata), + } + + if "error" in payload: + self.context.response.success = False + self.context.response.answer = f"Error: {payload['error']}" + else: + self.context.response.success = True + self.context.response.answer = f"Deleted {len(payload['deleted'])} key(s) from {path}" + self.context.response.metadata.update(payload) diff --git a/reme4/steps/frontmatter/read.py b/reme4/steps/frontmatter/read.py new file mode 100644 index 00000000..c27fb117 --- /dev/null +++ b/reme4/steps/frontmatter/read.py @@ -0,0 +1,44 @@ +"""``frontmatter_read_step`` — return the frontmatter dict of a markdown file. + +Cheap structured read — frontmatter only, no body. Use ``body:read`` +for the post-frontmatter content slice, or whole-file ``read`` when +you want both at once. Returns ``{exists: false}`` when the target +doesn't exist; otherwise ``{exists: true, frontmatter: {...}}``. + +``path`` is a path relative to the vault. +""" + +from pathlib import Path + +import frontmatter + +from ..base_step import BaseStep + +from ...components import R + + +@R.register("frontmatter_read_step") +class FrontmatterReadStep(BaseStep): + """Read a markdown file's frontmatter (YAML metadata only).""" + + async def execute(self): + assert self.context is not None + path: str = self.context.get("path") or "" + assert path, "path is required" + + target = (Path(self.file_store.vault_path or ".") / path).resolve() + if not target.is_file(): + self.context.response.success = False + self.context.response.answer = f"Error: {path} not found" + self.context.response.metadata.update({"path": path, "exists": False}) + return + if target.suffix != ".md": + self.context.response.success = False + self.context.response.answer = "Error: not markdown" + self.context.response.metadata.update({"path": path, "error": "not markdown"}) + return + + meta = dict(frontmatter.loads(target.read_text(encoding="utf-8")).metadata) + self.context.response.success = True + self.context.response.answer = f"Read frontmatter from {path} ({len(meta)} key(s))" + self.context.response.metadata.update({"path": path, "exists": True, "frontmatter": meta}) diff --git a/reme4/steps/frontmatter/update.py b/reme4/steps/frontmatter/update.py new file mode 100644 index 00000000..c00a7f89 --- /dev/null +++ b/reme4/steps/frontmatter/update.py @@ -0,0 +1,54 @@ +"""``frontmatter_update_step`` — set frontmatter keys on a markdown file. + +Read-modify-write the YAML frontmatter; body content is untouched. +The watcher / parser pick up the change asynchronously. + +Input shape: ``frontmatter_update_step path=foo.md metadata={"x": "y", "z": "w"}`` +— ``metadata`` is an explicit dict whose entries are merged into the +file's frontmatter (existing keys overwritten, missing keys inserted). + +``path`` is a path relative to the vault. Non-markdown targets return +``error="not markdown"``. An empty or missing ``metadata`` returns +``error="no fields to update"``. +""" + +from pathlib import Path + +import frontmatter + +from ..base_step import BaseStep + +from ...components import R + + +@R.register("frontmatter_update_step") +class FrontmatterUpdateStep(BaseStep): + """Set frontmatter keys on a markdown file from a ``metadata`` dict.""" + + async def execute(self): + assert self.context is not None + path: str = self.context.get("path") or "" + assert path, "path is required" + metadata = self.context.get("metadata") or {} + assert isinstance(metadata, dict), "metadata must be a dict" + + target = (Path(self.file_store.vault_path or ".") / path).resolve() + if not target.is_file(): + payload: dict = {"path": path, "error": "not found"} + elif target.suffix != ".md": + payload = {"path": path, "error": "not markdown"} + elif not metadata: + payload = {"path": path, "error": "no fields to update"} + else: + post = frontmatter.loads(target.read_text(encoding="utf-8")) + post.metadata.update(metadata) + target.write_text(frontmatter.dumps(post), encoding="utf-8") + payload = {"path": path, "updated": metadata} + + if "error" in payload: + self.context.response.success = False + self.context.response.answer = f"Error: {payload['error']}" + else: + self.context.response.success = True + self.context.response.answer = f"Updated {len(metadata)} key(s) on {path}" + self.context.response.metadata.update(payload) diff --git a/reme4/steps/graph/__init__.py b/reme4/steps/graph/__init__.py new file mode 100644 index 00000000..c5147dce --- /dev/null +++ b/reme4/steps/graph/__init__.py @@ -0,0 +1,7 @@ +"""Graph steps.""" + +from .traverse import GraphTraverseStep + +__all__ = [ + "GraphTraverseStep", +] diff --git a/reme4/steps/graph/traverse.py b/reme4/steps/graph/traverse.py new file mode 100644 index 00000000..25326f56 --- /dev/null +++ b/reme4/steps/graph/traverse.py @@ -0,0 +1,119 @@ +"""``graph_traverse_step`` — BFS over wikilink edges from a seed file. + +Single tool for relationship browsing. ``depth=1`` covers the trivial +"what does this link to / what links here" lookups (set ``direction`` +accordingly); higher depth opens up multi-hop exploration. + +Output is one record per edge traversed (not per node), so the same +target can appear multiple times if reached via different predicates +or paths — agents dedupe at the call site if they want a flat node +set. Each record carries ``via`` (the predecessor) and the link's +``predicate`` / ``anchor`` so the agent can reconstruct the path. + +Adjacency is loaded once via ``file_graph.get_nodes(None)`` — every +real node arrives with its full ``links`` payload, and we build both +the outbound and the inbound index in a single pass. The BFS then +runs purely in memory: no per-frontier-node graph round-trips, no +filesystem walk. The ``get_inlinks`` / ``get_outlinks`` contract +methods stay unused here because they'd add network round-trips for +data we already have. + +Direction vocabulary accepts both the standard convention +(``forward`` / ``backward`` / ``both``) and the engine convention +(``out`` / ``in`` / ``both``). + +The seed ``path`` is taken as-is (vault-relative). A seed that doesn't +match any graph node yields an empty result (no error). +""" + +from collections import deque + +from ..base_step import BaseStep +from ...components import R +from ...schema import FileLink + + +_FORWARD = {"out", "forward"} +_BACKWARD = {"in", "backward"} +_BOTH = {"both"} +_VALID_DIRECTIONS = _FORWARD | _BACKWARD | _BOTH + + +@R.register("graph_traverse_step") +class GraphTraverseStep(BaseStep): + """BFS from a seed file to explore wikilink relationships. + + Parameters: + path — seed path (vault-relative). + direction — ``forward`` / ``backward`` / ``both`` (or ``out`` / ``in`` / ``both``). + depth — hop limit (default 1 = immediate neighbors). + predicate — optional edge-type filter; ``None`` = no filter. + """ + + async def execute(self): + """BFS from ``path`` and emit one record per traversed edge.""" + assert self.context is not None + seed = str(self.context.get("path") or "").strip() + assert seed, "path is required" + max_depth = int(self.context.get("depth") or 1) + direction = (self.context.get("direction") or "both").lower() + predicate = self.context.get("predicate") + assert ( + direction in _VALID_DIRECTIONS + ), f"direction must be one of {sorted(_VALID_DIRECTIONS)}, got {direction!r}" + + # Build outbound / inbound adjacency in one pass over all nodes. + outbound: dict[str, list[tuple[str, FileLink]]] = {} + inbound: dict[str, list[tuple[str, FileLink]]] = {} + if self.file_store.file_graph: + for node in await self.file_store.file_graph.get_nodes(): + for link in node.links: + if not link.target_path: + continue + outbound.setdefault(node.path, []).append((link.target_path, link)) + inbound.setdefault(link.target_path, []).append((node.path, link)) + + walk_out = direction in _FORWARD or direction in _BOTH + walk_in = direction in _BACKWARD or direction in _BOTH + + visited_edges: set[tuple[str, str, str | None]] = set() + results: list[dict] = [] + queue: deque[tuple[str, int]] = deque([(seed, 0)]) + + while queue: + current, depth = queue.popleft() + if depth >= max_depth: + continue + + edges: list[tuple[str, str | None, str | None]] = [] + if walk_out: + for tgt, link in outbound.get(current, ()): + if predicate is not None and link.predicate != predicate: + continue + edges.append((tgt, link.predicate, link.target_anchor)) + if walk_in: + for src, link in inbound.get(current, ()): + if predicate is not None and link.predicate != predicate: + continue + edges.append((src, link.predicate, link.target_anchor)) + + for next_path, pred, anchor in edges: + edge_key = (current, next_path, pred) + if edge_key in visited_edges: + continue + visited_edges.add(edge_key) + results.append( + { + "path": next_path, + "depth": depth + 1, + "via": current, + "predicate": pred, + "anchor": anchor, + }, + ) + if depth + 1 < max_depth: + queue.append((next_path, depth + 1)) + + self.context.response.success = True + self.context.response.answer = f"Traversed {len(results)} edge(s) from {seed}" + self.context.response.metadata.update({"edges": results, "count": len(results)}) diff --git a/reme4/utils/common_utils.py b/reme4/utils/common_utils.py index 8c83535b..08fbc861 100644 --- a/reme4/utils/common_utils.py +++ b/reme4/utils/common_utils.py @@ -1,4 +1,4 @@ -"""Common utilities: hashing and async stream task execution.""" +"""Common utilities: hashing, async stream task execution, HTTP helpers.""" import asyncio import hashlib diff --git a/reme4/utils/wikilink_handler.py b/reme4/utils/wikilink_handler.py new file mode 100644 index 00000000..fddfb148 --- /dev/null +++ b/reme4/utils/wikilink_handler.py @@ -0,0 +1,387 @@ +"""Wikilink handler — single source of truth for ``[[...]]`` syntax. + +One class, :class:`WikilinkHandler`, owning every wikilink concern: + +* **Pure text** — regex, Dataview predicate inference, validation: + :meth:`~WikilinkHandler.extract_links` (used by + :mod:`reme4.components.file_parser.linked_file_parser`), + :meth:`~WikilinkHandler.scan_and_rewrite`, + :meth:`~WikilinkHandler.validate_src_dst` / + :meth:`~WikilinkHandler.validate_scope` / + :meth:`~WikilinkHandler.within_scope`. +* **Async, file_graph-aware** — + :meth:`~WikilinkHandler.find_inbound` (called by ``file_delete`` + to surface references the caller might want to clean up) and + :meth:`~WikilinkHandler.retarget_links` (called by ``file_move`` + post-rename to point inbound ``[[src]]`` at the new path). Source + candidates come from the file_graph's reverse index — no fs scan. + +Wikilink convention. Targets are taken **literally** — ``[[X]]`` → +``target="X"``, no implicit ``.md``, no short-form basename search, +no folder-note expansion. Anchor and alias survive a rewrite +verbatim. Image marker (``!``) and Dataview predicate (``pred::`` +outside the brackets) sit outside ``[[...]]`` and are not touched by +a rewrite. Recommended form: full path relative to the vault with +extension (``[[topics/x.md]]``). + +Stale graph entries are harmless (``scan_and_rewrite`` returns +count=0 and the file is skipped), but a graph missing recent writes +will miss those sources — keep the watcher in sync. +""" + +import re +from dataclasses import dataclass +from pathlib import Path + +from ..enumeration import LinkScopeEnum +from ..schema import FileLink + + +@dataclass(frozen=True) +class WikilinkMatch: + """One ``[[...]]`` occurrence with parts surfaced. + + ``anchor`` / ``alias`` are stored **without** the leading ``#`` / + ``|`` so they map cleanly to :class:`FileLink.target_anchor`; the + rewrite path reads the raw regex groups (with delimiters) directly + and doesn't go through this dataclass. + """ + + target: str + anchor: str | None + alias: str | None + bang: bool + start: int + end: int + + +class WikilinkHandler: + """Pure-text wikilink operations: parse, extract, rewrite, validate.""" + + # Captures: optional image marker (``!``), the bare target, an + # optional ``#anchor`` slice (with ``#``), and an optional ``|alias`` + # slice (with ``|``). The anchor / alias inner classes exclude ``[`` + # defensively so a runaway match on malformed input can't swallow + # following links. + WIKILINK_RE = re.compile( + r""" + (?P!?) + \[\[ + (?P[^\[\]\|\#\n]+?) + (?P\#[^\[\]\|\n]+)? + (?P\|[^\[\]\n]+)? + \]\] + """, + re.VERBOSE, + ) + + FORBIDDEN_IN_NEW = ("[", "]", "#", "|", "\n", "\r") + + _DATAVIEW_LINE_RE = re.compile( + r"^[ \t]*(?:[-*+][ \t]+)?(?P[A-Za-z][A-Za-z0-9_]*)\s*::\s*(?P.+?)\s*$", + re.MULTILINE, + ) + + _INLINE_FIELD_OPEN_RE = re.compile(r"\[(?P[A-Za-z][A-Za-z0-9_]*)\s*::\s*") + + # -- Low-level scan ------------------------------------------------ + + @classmethod + def iter_matches(cls, text: str): + """Yield :class:`WikilinkMatch` for every ``[[...]]`` in ``text``. + + Skips matches whose target is empty after strip (defensive). + """ + for m in cls.WIKILINK_RE.finditer(text): + target = m.group("target").strip() + if not target: + continue + anchor_raw = m.group("anchor") + alias_raw = m.group("alias") + yield WikilinkMatch( + target=target, + anchor=anchor_raw[1:].strip() if anchor_raw else None, + alias=alias_raw[1:].strip() if alias_raw else None, + bang=bool(m.group("bang")), + start=m.start(), + end=m.end(), + ) + + # -- FileLink extraction (with predicate inference) --------------- + + @classmethod + def extract_links(cls, text: str, source_path: str) -> list[FileLink]: + """Emit :class:`FileLink` edges for every wikilink in ``text``. + + No resolution: ``target_path`` is the bracket contents verbatim. + Results are deduped by ``(target_path, predicate, target_anchor)`` + preserving order. + """ + if not text: + return [] + inline_spans = cls._iter_inline_fields(text) + out: list[FileLink] = [] + seen: set[tuple] = set() + for wm in cls.iter_matches(text): + predicate = cls._predicate_for(text, wm.start, inline_spans) + key = (wm.target, predicate, wm.anchor) + if key in seen: + continue + seen.add(key) + out.append( + FileLink( + source_path=source_path, + target_path=wm.target, + target_anchor=wm.anchor, + predicate=predicate, + ), + ) + return out + + # -- Find / rewrite by literal target match ------------------------ + + @classmethod + def scan_and_rewrite( + cls, + text: str, + old: str, + new: str | None, + ) -> tuple[str, int]: + """Find (and optionally rewrite) wikilinks whose target equals ``old``. + + Returns ``(new_text, count)``. When ``new`` is ``None`` no rewrite + happens (the original text is returned), but the count is still + populated — used by ``find_inbound``. Matching is literal: + ``target == old``. No short-link, no implicit ``.md``, no + folder-note expansion. + """ + count = 0 + + def sub(match: re.Match) -> str: + nonlocal count + target = match.group("target").strip() + if target != old: + return match.group(0) + count += 1 + if new is None: + return match.group(0) + anchor = match.group("anchor") or "" + alias = match.group("alias") or "" + bang = match.group("bang") or "" + return f"{bang}[[{new}{anchor}{alias}]]" + + new_text = cls.WIKILINK_RE.sub(sub, text) + return new_text, count + + # -- Validation ---------------------------------------------------- + + @classmethod + def validate_src_dst(cls, src: str, dst: str) -> str | None: + """Return an error message for bad rewrite inputs, or None when OK.""" + if not src or not dst: + return "src and dst are required" + if any(ch in dst for ch in cls.FORBIDDEN_IN_NEW): + return "dst must not contain [ ] # | newline" + if Path(src).is_absolute() or Path(dst).is_absolute(): + return "src and dst must be relative to the vault" + return None + + @staticmethod + def validate_scope(scope: str) -> str | None: + """Return an error message for a bad scope, or None when OK.""" + if scope and Path(scope).is_absolute(): + return "scope must be relative to the vault" + return None + + @staticmethod + def within_scope(rel: str, scope: str) -> bool: + """``rel`` (relative to the vault) is inside ``scope`` (empty = anywhere).""" + if not scope: + return True + prefix = scope.rstrip("/") + "/" + return rel == scope or rel.startswith(prefix) + + # -- Predicate helpers (internal) --------------------------------- + + @classmethod + def _iter_inline_fields(cls, text: str) -> list[tuple[int, int, str]]: + """Find inline-bracketed ``[predicate:: …]`` field spans by depth scan.""" + out: list[tuple[int, int, str]] = [] + for m in cls._INLINE_FIELD_OPEN_RE.finditer(text): + depth = 1 + i = m.end() + n = len(text) + while i < n: + c = text[i] + if c == "\n": + break + if c == "[": + depth += 1 + elif c == "]": + depth -= 1 + if depth == 0: + out.append((m.start(), i + 1, m.group("predicate"))) + break + i += 1 + return out + + @classmethod + def _predicate_for( + cls, + text: str, + pos: int, + inline_spans: list[tuple[int, int, str]], + ) -> str | None: + """Resolve the predicate governing a wikilink at offset ``pos``. + + Precedence: inline-bracketed > line-level Dataview > none. + """ + for field_start, field_end, predicate in inline_spans: + if field_start <= pos < field_end: + return predicate + line_start = text.rfind("\n", 0, pos) + 1 + line_end = text.find("\n", pos) + if line_end == -1: + line_end = len(text) + m = cls._DATAVIEW_LINE_RE.match(text[line_start:line_end]) + if m and line_start + m.start("value") <= pos: + return m.group("predicate") + return None + + # -- Async file_graph-aware operations ----------------------------- + + @classmethod + async def _inbound_sources(cls, file_store, target: str) -> list[str]: + """Source paths the file_graph reports as referencing ``target``. + + Reverse-index lookup via ``file_graph.get_inlinks(target, scope=ALL)`` — + ``target`` is typically virtual here (the move/delete callers query for + references to a path that has just been removed), so ``scope=ALL`` is + required to surface sources whose edges sit in the pending bucket. + Each returned ``FileLink`` carries the linking node's ``source_path``; + we dedupe to a sorted list since one source can host multiple edges + (different anchor/predicate) to the same target. Returns ``[]`` when + there is no file_graph attached or no source references the target. + """ + if not file_store.file_graph: + return [] + inlinks = await file_store.file_graph.get_inlinks(target, scope=LinkScopeEnum.ALL) + return sorted({link.source_path for link in inlinks if link.source_path}) + + @classmethod + async def find_inbound(cls, file_store, target: str, scope: str = "") -> dict: + """Count wikilinks across the vault that point at ``target``. + + Literal matching: ``[[target]]`` only. The target file itself is + excluded — self-references don't survive a delete and aren't + actionable for the caller. Sources come from the file_graph's + reverse index; per-file counts come from reading each candidate + source (the graph dedupes by ``(target, predicate, anchor)`` so + it can't count repeated bare-wikilink occurrences directly). + + Result shape:: + + { + "target": str, + "scope": str | None, + "files_touched": int, # number of OTHER files containing >=1 ref + "links_total": int, # total ref count across those files + "by_file": [{"path": str, "count": int}, ...], + } + + On bad inputs returns ``{"target": ..., "error": str}``. + """ + if not target: + return {"target": target, "error": "target is required"} + if Path(target).is_absolute(): + return {"target": target, "error": "target must be relative to the vault"} + err = cls.validate_scope(scope) + if err is not None: + return {"target": target, "error": err} + + vault_dir = Path(file_store.vault_path or ".").resolve() + by_file: list[dict] = [] + total = 0 + + for rel in await cls._inbound_sources(file_store, target): + if rel == target: + continue # self-references not actionable for delete cleanup + if not cls.within_scope(rel, scope): + continue + try: + text = (vault_dir / rel).read_text(encoding="utf-8") + except Exception: + continue + _, count = cls.scan_and_rewrite(text, old=target, new=None) + if count > 0: + by_file.append({"path": rel, "count": count}) + total += count + + return { + "target": target, + "scope": scope or None, + "files_touched": len(by_file), + "links_total": total, + "by_file": by_file, + } + + @classmethod + async def retarget_links( + cls, + file_store, + src: str, + dst: str, + scope: str = "", + dry_run: bool = False, + ) -> dict: + """Rewrite every wikilink pointing at ``src`` to point at ``dst``. + + Pure helper — called directly by ``file_move`` post-rename. Literal + matching only; candidate sources come from the file_graph's reverse + index. + """ + err = cls.validate_src_dst(src, dst) + if err is not None: + return {"src": src, "dst": dst, "error": err} + if src == dst: + return { + "src": src, + "dst": dst, + "scope": scope or None, + "dry_run": dry_run, + "files_touched": 0, + "links_changed": 0, + "by_file": [], + } + err = cls.validate_scope(scope) + if err is not None: + return {"src": src, "dst": dst, "error": err} + + vault_dir = Path(file_store.vault_path or ".").resolve() + by_file: list[dict] = [] + total_changes = 0 + + for rel in await cls._inbound_sources(file_store, src): + if not cls.within_scope(rel, scope): + continue + abs_path = vault_dir / rel + try: + text = abs_path.read_text(encoding="utf-8") + except Exception: + continue + new_text, count = cls.scan_and_rewrite(text, old=src, new=dst) + if count > 0: + by_file.append({"path": rel, "count": count}) + total_changes += count + if not dry_run: + abs_path.write_text(new_text, encoding="utf-8") + + return { + "src": src, + "dst": dst, + "scope": scope or None, + "dry_run": dry_run, + "files_touched": len(by_file), + "links_changed": total_changes, + "by_file": by_file, + } diff --git a/tests4/unittest/test_common_steps.py b/tests4/unittest/test_common_steps.py index 9eb47c91..4b573e35 100644 --- a/tests4/unittest/test_common_steps.py +++ b/tests4/unittest/test_common_steps.py @@ -1,14 +1,28 @@ -"""End-to-end tests for reme4 common steps: spawn `reme4 start`, drive via HTTP, -verify responses, then shut down. Each test uses an isolated cwd so the working_dir -(.reme by default) does not collide. +"""Tests for reme4 common steps. + +Two surfaces share this file: + +* **HTTP / MCP E2E tests** (top half) spawn ``reme4 start`` via + ``mock_reme_server`` and drive ``version`` / ``help`` / ``search`` / + ``init`` / ``demo`` over the wire. Each test uses an isolated cwd so + the vault (``.reme`` by default) does not collide. +* **Direct unit tests** (bottom half) exercise ``TraverseStep`` + (registered as ``traverse_step``) — BFS over wikilink edges from a + seed file, forward / backward / both — against a freshly built + ``LocalFileStore`` (embedding disabled). """ +# pylint: disable=protected-access + import asyncio import os import tempfile import warnings from reme4 import __version__ as REME_VERSION +from reme4.components.file_store import LocalFileStore +from reme4.schema import FileLink, FileNode +from reme4.steps.common import traverse as traverse_mod from reme4.utils import call_action, call_and_check, mock_reme_server warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba") @@ -36,9 +50,31 @@ def _run(coro): asyncio.run(coro) -# --------------------------------------------------------------------------- -# Individual job tests -# --------------------------------------------------------------------------- +def _node(path: str, links: list[tuple[str, str | None, str | None]] | None = None) -> FileNode: + """Build a FileNode with (target_path, target_anchor, predicate) outgoing edges.""" + return FileNode( + path=path, + st_mtime=1.0, + links=[FileLink(source_path=path, target_path=t, target_anchor=a, predicate=p) for t, a, p in (links or [])], + ) + + +async def _make_store(nodes: list[FileNode]) -> LocalFileStore: + """LocalFileStore seeded with the given graph nodes (no files on disk).""" + store = LocalFileStore(store_name="t", embedding_model="") + await store.start() + if nodes: + await store.file_graph.upsert_nodes(nodes) + return store + + +def _edges(step) -> list[dict]: + return step.context.response.metadata.get("edges", []) + + +# =========================================================================== +# HTTP / MCP E2E tests: version / help / search / init / demo +# =========================================================================== def test_version_job(): @@ -78,7 +114,7 @@ def test_help_job(): and r.get("success") is True and isinstance(r.get("answer"), str) and r.get("metadata", {}).get("job_count", 0) > 0 - and "help" not in r["answer"] + and "`help`" not in r["answer"] ), ) # Spot-check that a couple of known jobs appear in the listing. @@ -132,35 +168,214 @@ def test_search_job_missing_query(): _run(run()) -def test_demo_job(): - """demo job should echo back the normalized query and adjusted min_score.""" +# -- aggregate: reuse one server instance for all jobs ------------------- + + +def test_all_jobs_one_server(): + """Run every common job against a single shared server for efficiency.""" async def run(): with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): async with mock_reme_server() as (host, port): + # version await call_and_check( - "demo", + "version", host=host, port=port, - query=" Hello World ", - min_score=0.8, - validator=lambda r: ( - isinstance(r, dict) - and r.get("success") is True - and "hello world" in str(r.get("answer", "")) - and abs(r.get("metadata", {}).get("adjusted_min_score", 0) - 0.72) < 1e-6 + validator=lambda r: isinstance(r, dict) and r.get("answer") == REME_VERSION, + ) + # help + await call_and_check( + "help", + host=host, + port=port, + validator=lambda r: isinstance(r, dict) and r.get("metadata", {}).get("job_count", 0) > 0, + ) + # health_check + await call_and_check( + "health_check", + host=host, + port=port, + validator=lambda r: isinstance(r, dict) + and isinstance( + r.get("metadata", {}).get("health"), + dict, ), ) - print("✓ test_demo_job passed") + # search (empty store) + await call_and_check( + "search", + host=host, + port=port, + query="anything", + validator=lambda r: isinstance(r, dict) and r.get("success") is True, + ) + # reindex + await call_and_check( + "reindex", + host=host, + port=port, + validator=lambda r: isinstance(r, dict) and isinstance(r.get("metadata", {}).get("counts"), dict), + ) + print("✓ test_all_jobs_one_server passed") _run(run()) +# =========================================================================== +# Direct unit tests: TraverseStep +# (LocalFileStore, no HTTP server — BFS over wikilink edges) +# =========================================================================== + + +def test_traverse_forward_depth_1(): + """depth=1 forward returns direct outbound neighbors.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + store = await _make_store( + [ + _node("a.md", [("b.md", None, None), ("c.md", "intro", "ref")]), + _node("b.md"), + _node("c.md"), + ], + ) + step = traverse_mod.TraverseStep(file_store=store) + await step(path="a.md", direction="forward", depth=1) + results = _edges(step) + paths = {r["path"] for r in results} + assert paths == {"b.md", "c.md"} + # The 'ref' edge should report its predicate/anchor. + c_edge = next(r for r in results if r["path"] == "c.md") + assert c_edge["predicate"] == "ref" + assert c_edge["anchor"] == "intro" + assert c_edge["via"] == "a.md" + assert c_edge["depth"] == 1 + await store.close() + print("✓ test_traverse_forward_depth_1 passed") + + asyncio.run(run()) + + +def test_traverse_backward_returns_inlinks(): + """direction=backward walks inbound edges.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + store = await _make_store( + [ + _node("a.md", [("b.md", None, None)]), + _node("c.md", [("b.md", None, None)]), + _node("b.md"), + ], + ) + step = traverse_mod.TraverseStep(file_store=store) + await step(path="b.md", direction="backward", depth=1) + results = _edges(step) + assert {r["path"] for r in results} == {"a.md", "c.md"} + await store.close() + print("✓ test_traverse_backward_returns_inlinks passed") + + asyncio.run(run()) + + +def test_traverse_depth_2_expands(): + """depth=2 traverses one hop beyond direct neighbors.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + store = await _make_store( + [ + _node("a.md", [("b.md", None, None)]), + _node("b.md", [("c.md", None, None)]), + _node("c.md"), + ], + ) + step = traverse_mod.TraverseStep(file_store=store) + await step(path="a.md", direction="forward", depth=2) + results = _edges(step) + depth_map = {r["path"]: r["depth"] for r in results} + assert depth_map.get("b.md") == 1 + assert depth_map.get("c.md") == 2 + await store.close() + print("✓ test_traverse_depth_2_expands passed") + + asyncio.run(run()) + + +def test_traverse_short_seed_yields_empty(): + """A short (not relative to the vault) seed isn't resolved anymore — BFS simply + finds no edges from a path that doesn't match any graph node.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + store = await _make_store( + [ + _node("topics/Bob.md"), + _node("people/Bob.md"), + ], + ) + step = traverse_mod.TraverseStep(file_store=store) + await step(path="Bob", direction="forward", depth=1) + payload = _edges(step) + # No error, just empty results because "Bob" isn't a graph key. + assert payload == [] + await store.close() + print("✓ test_traverse_short_seed_yields_empty passed") + + asyncio.run(run()) + + +def test_traverse_not_found_seed(): + """A seed not in the graph returns an empty list (no error).""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + store = await _make_store([_node("a.md")]) + step = traverse_mod.TraverseStep(file_store=store) + await step(path="topics/ghost.md", direction="forward", depth=1) + payload = _edges(step) + assert payload == [] + await store.close() + print("✓ test_traverse_not_found_seed passed") + + asyncio.run(run()) + + +def test_traverse_both_directions(): + """direction=both walks out- and in-bound; depth=1 returns one hop in each direction.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + store = await _make_store( + [ + _node("upstream.md", [("center.md", None, None)]), + _node("center.md", [("downstream.md", None, None)]), + _node("downstream.md"), + ], + ) + step = traverse_mod.TraverseStep(file_store=store) + await step(path="center.md", direction="both", depth=1) + results = _edges(step) + assert {r["path"] for r in results} == {"upstream.md", "downstream.md"} + await store.close() + print("✓ test_traverse_both_directions passed") + + asyncio.run(run()) + + if __name__ == "__main__": print("\n=== reme4 common steps E2E tests ===") test_version_job() test_help_job() test_search_job_empty_store() test_search_job_missing_query() - test_demo_job() + test_all_jobs_one_server() + print("\n=== traverse step tests ===") + test_traverse_forward_depth_1() + test_traverse_backward_returns_inlinks() + test_traverse_depth_2_expands() + test_traverse_short_seed_yields_empty() + test_traverse_not_found_seed() + test_traverse_both_directions() print("\n所有测试通过!") diff --git a/tests4/unittest/test_crud_steps.py b/tests4/unittest/test_crud_steps.py new file mode 100644 index 00000000..7159b287 --- /dev/null +++ b/tests4/unittest/test_crud_steps.py @@ -0,0 +1,1554 @@ +# pylint: disable=too-many-lines +"""Tests for crud steps — the opaque-byte vault_dir surface plus the +text-content ops (``read`` / ``write`` / ``edit`` / ``append``). + +Two surfaces share this file: + +* **Direct unit tests** (top half) drive each step against a freshly + built ``LocalFileStore`` (embedding disabled, BM25 kept) with files + registered in the graph so retarget's reverse-index lookup finds + inbound edges. Covers ``stat`` / ``list`` / ``download`` / ``move`` + / ``delete``. +* **HTTP/MCP E2E tests** (bottom half) spawn ``reme4 start`` via + ``mock_reme_server`` and exercise ``read`` / ``write`` / ``edit`` / + ``append`` end-to-end, including non-md degraded mode + encoding + edge cases. + +Frontmatter-only ops live in ``test_frontmatter_steps.py``. The +``upload`` step is a passive resource-ingest entry point with its own +bucket semantics — tests for it live in ``test_resource_steps.py``. + +CLI rule for the HTTP half: ``path=`` is relative-only, rooted at the +reme vault. A bare path with no suffix auto-appends ``.md``; +non-``.md`` suffix is accepted in degraded mode. Absolute paths are +accepted with a warning. +""" + +# pylint: disable=protected-access,redefined-builtin + +import asyncio +import os +import tempfile +import warnings +from pathlib import Path + +from reme4.components.file_store import LocalFileStore +from reme4.schema import FileNode +from reme4.steps.crud import ( + delete as crud_delete, + download as crud_download, + list as crud_list, + move as crud_move, + stat as crud_stat, +) +from reme4.utils import call_action, call_and_check, mock_reme_server +from reme4.utils.wikilink_handler import WikilinkHandler + +warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources") + + +class temp_chdir: + """Context manager to temporarily chdir into a path and restore on exit.""" + + def __init__(self, path): + self.path = path + self.old = None + + def __enter__(self): + self.old = os.getcwd() + os.chdir(self.path) + return self + + def __exit__(self, *exc): + os.chdir(self.old) + + +async def _make_store(files: dict[str, str] | None = None) -> LocalFileStore: + """LocalFileStore seeded with files on disk + registered in the graph.""" + store = LocalFileStore(store_name="t", embedding_model="") + await store.start() + nodes: list[FileNode] = [] + for rel, content in (files or {}).items(): + abs_path = Path.cwd() / rel + abs_path.parent.mkdir(parents=True, exist_ok=True) + abs_path.write_text(content, encoding="utf-8") + nodes.append( + FileNode( + path=rel, + st_mtime=abs_path.stat().st_mtime, + links=WikilinkHandler.extract_links(content, rel), + ), + ) + if nodes: + await store.file_graph.upsert_nodes(nodes) + return store + + +def _metadata(step) -> dict: + return step.context.response.metadata + + +def _run(coro): + """Run an async coroutine on a fresh isolated event loop.""" + asyncio.run(coro) + + +def _seed_md(vault_dir: Path, rel: str, body: str) -> Path: + target = vault_dir / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body, encoding="utf-8") + return target + + +# =========================================================================== +# Direct unit tests: stat / list / download / move / delete +# (LocalFileStore, no HTTP server) +# =========================================================================== + + +# -- stat ---------------------------------------------------------------- + + +def test_stat_indexed_file(): + """stat returns size, mime, and frontmatter for an indexed .md file.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store({"topics/n.md": "---\nname: T\n---\nbody"}) + step = crud_stat.StatStep(file_store=store) + await step(path="topics/n.md") + payload = _metadata(step) + assert payload["exists"] is True + assert payload["type"] == "file" + assert "size" in payload and payload["size"] > 0 + assert payload["mime"].startswith("text/") + assert payload["frontmatter"] == {"name": "T"} + await store.close() + print("✓ test_stat_indexed_file passed") + + asyncio.run(run()) + + +def test_stat_directory_fallback(): + """stat on a non-indexed directory falls back to a plain join + type=dir.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store() + (Path(tmp) / "topics").mkdir(parents=True, exist_ok=True) + step = crud_stat.StatStep(file_store=store) + await step(path="topics") + payload = _metadata(step) + assert payload["exists"] is True + assert payload["type"] == "dir" + await store.close() + print("✓ test_stat_directory_fallback passed") + + asyncio.run(run()) + + +# -- list ---------------------------------------------------------------- + + +def test_list_lists_files(): + """list returns paths relative to the vault for files under the given directory.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store( + { + "topics/a.md": "x", + "topics/b.md": "y", + "topics/sub/c.md": "z", + }, + ) + step = crud_list.ListStep(file_store=store) + await step(path="topics", recursive=True) + payload = _metadata(step) + assert set(payload["items"]) == {"topics/a.md", "topics/b.md", "topics/sub/c.md"} + assert payload["count"] == 3 + await store.close() + print("✓ test_list_lists_files passed") + + asyncio.run(run()) + + +def test_list_respects_limit_and_non_recursive(): + """Non-recursive list ignores subdirs; limit caps the count.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store( + { + "topics/a.md": "x", + "topics/b.md": "y", + "topics/sub/c.md": "z", + }, + ) + step = crud_list.ListStep(file_store=store) + await step(path="topics", recursive=False, limit=1) + payload = _metadata(step) + assert payload["count"] == 1 + assert len(payload["items"]) == 1 + await store.close() + print("✓ test_list_respects_limit_and_non_recursive passed") + + asyncio.run(run()) + + +# -- download ------------------------------------------------------------ + + +def test_download_to_explicit_path(): + """download copies the vault file to dst_path.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store({"topics/a.md": "alpha"}) + target = Path(tmp) / "out" / "a.md" + step = crud_download.DownloadStep(file_store=store) + await step(src_path="topics/a.md", dst_path=str(target)) + payload = _metadata(step) + assert "error" not in payload + assert payload["dst_path"] == str(target) + assert target.read_text(encoding="utf-8") == "alpha" + await store.close() + print("✓ test_download_to_explicit_path passed") + + asyncio.run(run()) + + +def test_download_to_temp_when_dst_path_empty(): + """Without dst_path, download lands the file in a temp file and returns the path.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store({"topics/a.md": "alpha"}) + step = crud_download.DownloadStep(file_store=store) + await step(src_path="topics/a.md") + payload = _metadata(step) + assert "error" not in payload + assert Path(payload["dst_path"]).read_text(encoding="utf-8") == "alpha" + await store.close() + print("✓ test_download_to_temp_when_dst_path_empty passed") + + asyncio.run(run()) + + +# -- move ---------------------------------------------------------------- + + +def test_move_relocates_within_vault(): + """move renames / relocates a file in place.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store({"daily/2026-05-18/foo/foo.md": "draft"}) + step = crud_move.MoveStep(file_store=store) + await step(src_path="daily/2026-05-18/foo/foo.md", dst_path="knowledge/foo/foo.md") + payload = _metadata(step) + assert "error" not in payload + assert not (Path(tmp) / "daily/2026-05-18/foo/foo.md").exists() + assert (Path(tmp) / "knowledge/foo/foo.md").read_text(encoding="utf-8") == "draft" + await store.close() + print("✓ test_move_relocates_within_vault passed") + + asyncio.run(run()) + + +def test_move_refuses_overwrite_without_flag(): + """move refuses to clobber an existing dst_path unless overwrite=True.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store( + { + "a/x.md": "src", + "b/x.md": "dst", + }, + ) + step = crud_move.MoveStep(file_store=store) + await step(src_path="a/x.md", dst_path="b/x.md") + payload = _metadata(step) + assert "destination exists" in payload.get("error", "") + assert (Path(tmp) / "a/x.md").exists() + await store.close() + print("✓ test_move_refuses_overwrite_without_flag passed") + + asyncio.run(run()) + + +def test_move_default_retargets_inbound_links(): + """move with retarget=True (default) rewrites inbound full-path [[src_path]] → [[dst_path]].""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store( + { + "daily/2026-05-18/draft/draft.md": "spec", + # only the literal full-path form is retargeted by design; + # short / no-ext forms are intentionally left alone. + "knowledge/notes/notes.md": ( + "see [[daily/2026-05-18/draft/draft.md]] and again " "[[daily/2026-05-18/draft/draft.md]] twice" + ), + }, + ) + step = crud_move.MoveStep(file_store=store) + await step( + src_path="daily/2026-05-18/draft/draft.md", + dst_path="knowledge/draft/draft.md", + ) + payload = _metadata(step) + + assert "error" not in payload + assert payload["retarget"]["files_touched"] == 1 + assert payload["retarget"]["links_changed"] == 2 + + notes = (Path(tmp) / "knowledge/notes/notes.md").read_text(encoding="utf-8") + assert "[[daily/2026-05-18/draft/draft.md]]" not in notes + assert notes.count("[[knowledge/draft/draft.md]]") == 2 + await store.close() + print("✓ test_move_default_retargets_inbound_links passed") + + asyncio.run(run()) + + +def test_move_opt_out_leaves_links_dangling(): + """retarget=False moves the file but leaves inbound references stale.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store( + { + "daily/2026-05-18/draft/draft.md": "spec", + "knowledge/notes/notes.md": "see [[daily/2026-05-18/draft/draft.md]]", + }, + ) + step = crud_move.MoveStep(file_store=store) + await step( + src_path="daily/2026-05-18/draft/draft.md", + dst_path="knowledge/draft/draft.md", + retarget=False, + ) + payload = _metadata(step) + + assert "error" not in payload + assert payload["retarget"] is None + + notes = (Path(tmp) / "knowledge/notes/notes.md").read_text(encoding="utf-8") + # link UNCHANGED — caller opted out of retarget + assert "[[daily/2026-05-18/draft/draft.md]]" in notes + await store.close() + print("✓ test_move_opt_out_leaves_links_dangling passed") + + asyncio.run(run()) + + +# -- delete -------------------------------------------------------------- + + +def test_delete_removes_file(): + """delete hard-removes the file.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store({"knowledge/draft/draft.md": "x"}) + step = crud_delete.DeleteStep(file_store=store) + await step(path="knowledge/draft/draft.md") + payload = _metadata(step) + assert payload.get("deleted") is True + assert not (Path(tmp) / "knowledge/draft/draft.md").exists() + await store.close() + print("✓ test_delete_removes_file passed") + + asyncio.run(run()) + + +def test_delete_missing_returns_error(): + """delete on a nonexistent path returns error rather than raising.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store() + step = crud_delete.DeleteStep(file_store=store) + await step(path="knowledge/nope/nope.md") + payload = _metadata(step) + assert payload["error"] == "not found" + await store.close() + print("✓ test_delete_missing_returns_error passed") + + asyncio.run(run()) + + +def test_delete_reports_inbound_refs(): + """delete returns the inbound wikilink list (literal full-path matches only).""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store( + { + "knowledge/target/target.md": "doomed", + "knowledge/a/a.md": "see [[knowledge/target/target.md]]", + "knowledge/b/b.md": ( + "ref [[knowledge/target/target.md]] and again " "[[knowledge/target/target.md]]" + ), + # short / no-ext forms are NOT counted by design + "knowledge/c/c.md": "[[target]] and [[knowledge/target/target]]", + }, + ) + step = crud_delete.DeleteStep(file_store=store) + await step(path="knowledge/target/target.md") + payload = _metadata(step) + + assert payload["deleted"] is True + assert not (Path(tmp) / "knowledge/target/target.md").exists() + # referencing files are untouched — agent decides what to do + assert (Path(tmp) / "knowledge/a/a.md").read_text(encoding="utf-8") == ( + "see [[knowledge/target/target.md]]" + ) + + inbound = payload["inbound"] + paths = {item["path"] for item in inbound["by_file"]} + assert paths == {"knowledge/a/a.md", "knowledge/b/b.md"} + # a: 1 full-path ref; b: 2 full-path refs; c: not counted + assert inbound["files_touched"] == 2 + assert inbound["links_total"] == 3 + await store.close() + print("✓ test_delete_reports_inbound_refs passed") + + asyncio.run(run()) + + +def test_delete_folder_removes_tree(): + """delete on a directory hard-removes the whole subtree.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store( + { + "scratch/a.md": "alpha", + "scratch/sub/b.md": "beta", + "scratch/asset.bin": "blob", + "keeper/k.md": "kept", + }, + ) + step = crud_delete.DeleteStep(file_store=store) + await step(path="scratch") + payload = _metadata(step) + assert payload["deleted"] is True + assert payload["is_dir"] is True + assert set(payload["deleted_files"]) == { + "scratch/a.md", + "scratch/sub/b.md", + "scratch/asset.bin", + } + assert not (Path(tmp) / "scratch").exists() + assert (Path(tmp) / "keeper/k.md").exists() + await store.close() + print("✓ test_delete_folder_removes_tree passed") + + asyncio.run(run()) + + +def test_delete_folder_reports_only_external_inbound(): + """Inbound from inside the doomed folder is suppressed; outside refs surface.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store( + { + "doomed/a.md": "see [[doomed/b.md]]", # internal — filtered out + "doomed/b.md": "see [[doomed/a.md]]", # internal — filtered out + "outside/x.md": ("ext [[doomed/a.md]] and [[doomed/a.md]] plus [[doomed/b.md]]"), + "outside/y.md": "another [[doomed/a.md]]", + }, + ) + step = crud_delete.DeleteStep(file_store=store) + await step(path="doomed") + payload = _metadata(step) + assert payload["deleted"] is True + assert payload["is_dir"] is True + assert not (Path(tmp) / "doomed").exists() + # outside files survive untouched + assert (Path(tmp) / "outside/x.md").exists() + + inbound = payload["inbound"] + # external sources: x.md, y.md (deduped) → 2 files + assert inbound["files_touched"] == 2 + # 2 refs to doomed/a.md from x + 1 ref from y + 1 ref to b from x = 4 + assert inbound["links_total"] == 4 + + by_target = {row["target"]: row for row in inbound["by_target"]} + assert set(by_target) == {"doomed/a.md", "doomed/b.md"} + a_sources = {row["path"]: row["count"] for row in by_target["doomed/a.md"]["by_file"]} + assert a_sources == {"outside/x.md": 2, "outside/y.md": 1} + b_sources = {row["path"]: row["count"] for row in by_target["doomed/b.md"]["by_file"]} + assert b_sources == {"outside/x.md": 1} + await store.close() + print("✓ test_delete_folder_reports_only_external_inbound passed") + + asyncio.run(run()) + + +def test_delete_folder_empty_has_no_inbound(): + """Empty folder delete reports zero deleted files and zero inbound.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + (Path(tmp) / "empty").mkdir() + store = await _make_store() + step = crud_delete.DeleteStep(file_store=store) + await step(path="empty") + payload = _metadata(step) + assert payload["deleted"] is True + assert payload["is_dir"] is True + assert payload["deleted_files"] == [] + assert payload["inbound"]["files_touched"] == 0 + assert payload["inbound"]["links_total"] == 0 + assert not (Path(tmp) / "empty").exists() + await store.close() + print("✓ test_delete_folder_empty_has_no_inbound passed") + + asyncio.run(run()) + + +# =========================================================================== +# HTTP / MCP E2E tests: read / write / edit / append +# (mock_reme_server spawns `reme4 start`, calls go over the wire) +# =========================================================================== + + +# -- read ---------------------------------------------------------------- + + +def test_read_relative_path(): + """`reme4 read path=Templates/Recipe.md` returns the file body from vault/.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + body = "# Recipe\n\nMix flour and water.\n" + _seed_md(working, "Templates/Recipe.md", body) + async with mock_reme_server() as (host, port): + await call_and_check( + "read", + host=host, + port=port, + path="Templates/Recipe.md", + validator=lambda r: ( + isinstance(r, dict) + and r.get("success") is True + and "# Recipe" in str(r.get("answer", "")) + and "flour and water" in str(r.get("answer", "")) + ), + ) + print("✓ test_read_relative_path passed") + + _run(run()) + + +def test_read_no_suffix_autoappends_md(): + """A bare path with no suffix auto-appends `.md`.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "Templates/Recipe.md", "auto-md\n") + async with mock_reme_server() as (host, port): + await call_and_check( + "read", + host=host, + port=port, + path="Templates/Recipe", + validator=lambda r: ( + isinstance(r, dict) and r.get("success") is True and "auto-md" in str(r.get("answer", "")) + ), + ) + print("✓ test_read_no_suffix_autoappends_md passed") + + _run(run()) + + +def test_read_line_range(): + """start_line / end_line slice the file 1-based, inclusive.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "Notes.md", "L1\nL2\nL3\nL4\nL5\n") + async with mock_reme_server() as (host, port): + await call_and_check( + "read", + host=host, + port=port, + path="Notes.md", + start_line=2, + end_line=4, + validator=lambda r: ( + isinstance(r, dict) + and r.get("success") is True + and "L2" in str(r["answer"]) + and "L3" in str(r["answer"]) + and "L4" in str(r["answer"]) + and "L1" not in str(r["answer"]) + and "L5" not in str(r["answer"]) + ), + ) + print("✓ test_read_line_range passed") + + _run(run()) + + +def test_read_absolute_path_accepted(): + """Absolute paths are accepted (a log warning is emitted but the read proceeds).""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + target = _seed_md(working, "Abs.md", "x\n") + async with mock_reme_server() as (host, port): + result = await call_action( + "read", + host=host, + port=port, + path=str(target.resolve()), + ) + if not ( + isinstance(result, dict) and result.get("success") is True and "x" in str(result.get("answer", "")) + ): + raise AssertionError(f"expected absolute-path read to succeed, got {result!r}") + print("✓ test_read_absolute_path_accepted passed") + + _run(run()) + + +def test_read_non_md_degraded(): + """Paths whose suffix is not `.md` are read in compatibility mode.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "data/foo.txt", "plain-text body\n") + async with mock_reme_server() as (host, port): + await call_and_check( + "read", + host=host, + port=port, + path="data/foo.txt", + validator=lambda r: ( + isinstance(r, dict) + and r.get("success") is True + and "plain-text body" in str(r.get("answer", "")) + ), + ) + print("✓ test_read_non_md_degraded passed") + + _run(run()) + + +def test_read_missing_file(): + """Reading a non-existent file should fail with a clear error.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + async with mock_reme_server() as (host, port): + result = await call_action( + "read", + host=host, + port=port, + path="NotThere.md", + ) + if not ( + isinstance(result, dict) + and result.get("success") is False + and "does not exist" in str(result.get("answer", "")).lower() + ): + raise AssertionError(f"expected missing-file rejection, got {result!r}") + print("✓ test_read_missing_file passed") + + _run(run()) + + +def test_read_start_after_end(): + """start_line > end_line is invalid.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "Range.md", "a\nb\nc\n") + async with mock_reme_server() as (host, port): + result = await call_action( + "read", + host=host, + port=port, + path="Range.md", + start_line=3, + end_line=1, + ) + if not ( + isinstance(result, dict) + and result.get("success") is False + and "start_line" in str(result.get("answer", "")) + ): + raise AssertionError(f"expected start>end rejection, got {result!r}") + print("✓ test_read_start_after_end passed") + + _run(run()) + + +def test_read_start_line_exceeds_total(): + """start_line beyond total line count is invalid.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "Short.md", "only-one-line\n") + async with mock_reme_server() as (host, port): + result = await call_action( + "read", + host=host, + port=port, + path="Short.md", + start_line=99, + ) + if not ( + isinstance(result, dict) + and result.get("success") is False + and "exceeds" in str(result.get("answer", "")).lower() + ): + raise AssertionError(f"expected exceeds-length rejection, got {result!r}") + print("✓ test_read_start_line_exceeds_total passed") + + _run(run()) + + +def test_read_truncation(): + """A file larger than DEFAULT_MAX_BYTES triggers truncation with a continuation notice.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + # Seed > DEFAULT_MAX_BYTES (50 KiB) so the default truncation kicks in. + body = "\n".join(f"line {i}" for i in range(8000)) + "\n" + _seed_md(working, "Big.md", body) + async with mock_reme_server() as (host, port): + await call_and_check( + "read", + host=host, + port=port, + path="Big.md", + validator=lambda r: ( + isinstance(r, dict) + and r.get("success") is True + and "truncated" in str(r["answer"]) + and "start_line=" in str(r["answer"]) + ), + ) + print("✓ test_read_truncation passed") + + _run(run()) + + +def test_read_empty_path_rejected(): + """An empty `path` should be rejected.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + async with mock_reme_server() as (host, port): + result = await call_action("read", host=host, port=port, path="") + if not ( + isinstance(result, dict) + and result.get("success") is False + and "required" in str(result.get("answer", "")).lower() + ): + raise AssertionError(f"expected `path` required rejection, got {result!r}") + print("✓ test_read_empty_path_rejected passed") + + _run(run()) + + +# -- write / edit / append ----------------------------------------------- + + +def test_write_basic_with_frontmatter(): + """`reme4 write path=... name=... description=... content=...` writes a YAML front matter block.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + async with mock_reme_server() as (host, port): + await call_and_check( + "write", + host=host, + port=port, + path="Notes/A.md", + name="Greetings", + description="a friendly hello note", + content="# Hello", + validator=lambda r: ( + isinstance(r, dict) and r.get("success") is True and "Wrote" in str(r.get("answer", "")) + ), + ) + on_disk = (working / "Notes/A.md").read_text(encoding="utf-8") + assert on_disk.startswith("---\n"), on_disk + assert "name: Greetings" in on_disk + assert "description: a friendly hello note" in on_disk + assert "# Hello" in on_disk + print("✓ test_write_basic_with_frontmatter passed") + + _run(run()) + + +def test_write_no_suffix_autoappends_md(): + """`path` with no suffix gets `.md` appended.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + async with mock_reme_server() as (host, port): + await call_and_check( + "write", + host=host, + port=port, + path="Notes/My", + content="x", + validator=lambda r: r.get("success") is True, + ) + assert (working / "Notes/My.md").exists() + print("✓ test_write_no_suffix_autoappends_md passed") + + _run(run()) + + +def test_write_overwrites_with_notice(): + """Writing into an existing path overwrites the file and surfaces a system notice.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "Existing.md", "old\n") + async with mock_reme_server() as (host, port): + await call_and_check( + "write", + host=host, + port=port, + path="Existing.md", + content="new", + validator=lambda r: ( + isinstance(r, dict) + and r.get("success") is True + and "Wrote" in str(r.get("answer", "")) + and "already existed" in str(r.get("answer", "")) + and "overwritten" in str(r.get("answer", "")) + ), + ) + # File body has been replaced. + on_disk = (working / "Existing.md").read_text(encoding="utf-8") + assert "new" in on_disk and "old" not in on_disk, on_disk + print("✓ test_write_overwrites_with_notice passed") + + _run(run()) + + +def test_write_creates_parent_dirs(): + """Nested-non-existent parents are auto-created.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + async with mock_reme_server() as (host, port): + await call_and_check( + "write", + host=host, + port=port, + path="a/b/c/D.md", + content="hi", + validator=lambda r: r.get("success") is True, + ) + assert (working / "a/b/c/D.md").exists() + print("✓ test_write_creates_parent_dirs passed") + + _run(run()) + + +def test_write_no_frontmatter_when_all_empty(): + """When both `name` and `description` are empty strings, the file is body-only. + + The CLI schema declares them required, but the step is intentionally lenient + so manual calls without these fields don't fail catastrophically.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + async with mock_reme_server() as (host, port): + await call_and_check( + "write", + host=host, + port=port, + path="Plain.md", + name="", + description="", + content="# Hello", + validator=lambda r: r.get("success") is True, + ) + on_disk = (working / "Plain.md").read_text(encoding="utf-8") + assert not on_disk.startswith("---"), on_disk + assert "# Hello" in on_disk + print("✓ test_write_no_frontmatter_when_all_empty passed") + + _run(run()) + + +def test_write_ignores_arbitrary_extra_fields(): + """Extra kwargs beyond name/description are silently ignored (schema is strict).""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + async with mock_reme_server() as (host, port): + await call_and_check( + "write", + host=host, + port=port, + path="Custom.md", + name="My Note", + description="short summary", + content="body", + # Extras below should NOT appear in front matter under the + # hardcoded-fields schema. + title="ignored", + author="ignored", + tags='["x","y"]', + validator=lambda r: r.get("success") is True, + ) + on_disk = (working / "Custom.md").read_text(encoding="utf-8") + assert on_disk.startswith("---\n"), on_disk + assert "name: My Note" in on_disk + assert "description: short summary" in on_disk + assert "title:" not in on_disk + assert "author:" not in on_disk + assert "tags:" not in on_disk + assert "body" in on_disk + print("✓ test_write_ignores_arbitrary_extra_fields passed") + + _run(run()) + + +def test_write_only_description_present(): + """Step is lenient: providing only `description` works; missing `name` is skipped.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + async with mock_reme_server() as (host, port): + await call_and_check( + "write", + host=host, + port=port, + path="OnlyDesc.md", + description="just a description", + content="body", + validator=lambda r: r.get("success") is True, + ) + on_disk = (working / "OnlyDesc.md").read_text(encoding="utf-8") + assert on_disk.startswith("---\n"), on_disk + assert "description: just a description" in on_disk + assert "name:" not in on_disk + print("✓ test_write_only_description_present passed") + + _run(run()) + + +def test_edit_global_replace(): + """`reme4 edit` replaces every occurrence of `old` with `new`.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "E.md", "foo bar foo\nfoo\n") + async with mock_reme_server() as (host, port): + await call_and_check( + "edit", + host=host, + port=port, + path="E.md", + old="foo", + new="qux", + validator=lambda r: ( + r.get("success") is True and "3" in str(r.get("answer", "")) # 3 replacements + ), + ) + assert (working / "E.md").read_text(encoding="utf-8") == "qux bar qux\nqux\n" + print("✓ test_edit_global_replace passed") + + _run(run()) + + +def test_edit_old_not_found(): + """`old` absent in the file → success=False.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "E.md", "hello world\n") + async with mock_reme_server() as (host, port): + result = await call_action( + "edit", + host=host, + port=port, + path="E.md", + old="absent", + new="x", + ) + if not ( + isinstance(result, dict) + and result.get("success") is False + and "not found" in str(result.get("answer", "")).lower() + ): + raise AssertionError(f"expected not-found rejection, got {result!r}") + # File unchanged. + assert (working / "E.md").read_text(encoding="utf-8") == "hello world\n" + print("✓ test_edit_old_not_found passed") + + _run(run()) + + +def test_edit_missing_file(): + """Editing a non-existent file should fail.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + async with mock_reme_server() as (host, port): + result = await call_action( + "edit", + host=host, + port=port, + path="NotThere.md", + old="x", + new="y", + ) + if not ( + isinstance(result, dict) + and result.get("success") is False + and "does not exist" in str(result.get("answer", "")).lower() + ): + raise AssertionError(f"expected missing-file rejection, got {result!r}") + print("✓ test_edit_missing_file passed") + + _run(run()) + + +def test_edit_skips_frontmatter(): + """A match present in both front matter and body is replaced only in the body.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + body = ( + "---\n" + "name: alpha\n" + "description: alpha-doc\n" + "---\n" + "intro paragraph mentioning alpha and alpha again.\n" + ) + _seed_md(working, "WithFM.md", body) + async with mock_reme_server() as (host, port): + await call_and_check( + "edit", + host=host, + port=port, + path="WithFM.md", + old="alpha", + new="beta", + validator=lambda r: ( + r.get("success") is True and "2" in str(r.get("answer", "")) # 2 body occurrences only + ), + ) + on_disk = (working / "WithFM.md").read_text(encoding="utf-8") + # Front matter untouched. + assert "name: alpha" in on_disk, on_disk + assert "description: alpha-doc" in on_disk, on_disk + # Body fully rewritten. + assert "beta and beta" in on_disk, on_disk + assert "alpha and alpha" not in on_disk, on_disk + print("✓ test_edit_skips_frontmatter passed") + + _run(run()) + + +def test_edit_match_only_in_frontmatter_fails(): + """If `old` appears ONLY inside front matter, edit reports not-found and writes nothing.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + body = "---\nname: secret\ndescription: nope\n---\nplain body without the keyword.\n" + _seed_md(working, "FMOnly.md", body) + async with mock_reme_server() as (host, port): + result = await call_action( + "edit", + host=host, + port=port, + path="FMOnly.md", + old="secret", + new="leaked", + ) + if not ( + isinstance(result, dict) + and result.get("success") is False + and "not found" in str(result.get("answer", "")).lower() + ): + raise AssertionError(f"expected not-found rejection, got {result!r}") + # File untouched. + assert (working / "FMOnly.md").read_text(encoding="utf-8") == body + print("✓ test_edit_match_only_in_frontmatter_fails passed") + + _run(run()) + + +def test_append_basic(): + """Append adds content to the end of an existing file.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "A.md", "L1\n") + async with mock_reme_server() as (host, port): + await call_and_check( + "append", + host=host, + port=port, + path="A.md", + content="L2\n", + validator=lambda r: r.get("success") is True and "Appended" in r["answer"], + ) + assert (working / "A.md").read_text(encoding="utf-8") == "L1\nL2\n" + print("✓ test_append_basic passed") + + _run(run()) + + +def test_append_concatenates_verbatim(): + """Append concatenates content verbatim — no implicit newline insertion.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "A.md", "abc") # no trailing newline + async with mock_reme_server() as (host, port): + await call_and_check( + "append", + host=host, + port=port, + path="A.md", + content="def", + validator=lambda r: r.get("success") is True, + ) + assert (working / "A.md").read_text(encoding="utf-8") == "abcdef" + print("✓ test_append_concatenates_verbatim passed") + + _run(run()) + + +def test_append_auto_creates_missing_file(): + """Append on a non-existent path creates the file and surfaces a system notice.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + async with mock_reme_server() as (host, port): + await call_and_check( + "append", + host=host, + port=port, + path="Fresh.md", + content="hello\n", + validator=lambda r: ( + isinstance(r, dict) + and r.get("success") is True + and "Appended" in str(r.get("answer", "")) + and "auto-created" in str(r.get("answer", "")) + ), + ) + assert (working / "Fresh.md").read_text(encoding="utf-8") == "hello\n" + print("✓ test_append_auto_creates_missing_file passed") + + _run(run()) + + +def test_append_empty_content_on_existing_file_is_noop(): + """Appending empty content to an existing file leaves it unchanged.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "A.md", "L1\n") + async with mock_reme_server() as (host, port): + await call_and_check( + "append", + host=host, + port=port, + path="A.md", + content="", + validator=lambda r: r.get("success") is True and "0 bytes" in str(r.get("answer", "")), + ) + assert (working / "A.md").read_text(encoding="utf-8") == "L1\n" + print("✓ test_append_empty_content_on_existing_file_is_noop passed") + + _run(run()) + + +def test_append_empty_content_creates_empty_file(): + """Appending empty content to a missing path creates an empty file (with notice).""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + async with mock_reme_server() as (host, port): + await call_and_check( + "append", + host=host, + port=port, + path="Empty.md", + content="", + validator=lambda r: (r.get("success") is True and "auto-created" in str(r.get("answer", ""))), + ) + target = working / "Empty.md" + assert target.exists() and target.read_text(encoding="utf-8") == "" + print("✓ test_append_empty_content_creates_empty_file passed") + + _run(run()) + + +# -- non-markdown degraded mode + encoding edge cases -------------------- + + +def test_write_non_md_skips_frontmatter(): + """Writing to a non-md path skips name/description and emits a recommendation notice.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + async with mock_reme_server() as (host, port): + await call_and_check( + "write", + host=host, + port=port, + path="data/notes.txt", + name="Greetings", + description="should be ignored", + content="# Hello", + validator=lambda r: ( + r.get("success") is True + and "Wrote" in str(r.get("answer", "")) + and "non-markdown" in str(r.get("answer", "")).lower() + ), + ) + on_disk = (working / "data/notes.txt").read_text(encoding="utf-8") + assert not on_disk.startswith("---"), on_disk + assert "name: Greetings" not in on_disk + assert on_disk == "# Hello" + print("✓ test_write_non_md_skips_frontmatter passed") + + _run(run()) + + +def test_edit_non_md_full_text(): + """Editing a non-md path operates on the full file body (no frontmatter parsing).""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + # A YAML-looking header that would otherwise be stripped as frontmatter. + body = "---\nname: keep-me\n---\nfoo bar foo\n" + _seed_md(working, "data/code.txt", body) + async with mock_reme_server() as (host, port): + await call_and_check( + "edit", + host=host, + port=port, + path="data/code.txt", + old="keep-me", + new="replaced", + validator=lambda r: ( + r.get("success") is True + and "1" in str(r.get("answer", "")) + and "non-markdown" in str(r.get("answer", "")).lower() + ), + ) + on_disk = (working / "data/code.txt").read_text(encoding="utf-8") + assert "name: replaced" in on_disk, on_disk + assert "foo bar foo" in on_disk, on_disk + print("✓ test_edit_non_md_full_text passed") + + _run(run()) + + +def test_append_non_md_warns(): + """Appending to a non-md file succeeds and surfaces the compatibility notice.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "data/log.txt", "line1\n") + async with mock_reme_server() as (host, port): + await call_and_check( + "append", + host=host, + port=port, + path="data/log.txt", + content="line2\n", + validator=lambda r: ( + r.get("success") is True and "non-markdown" in str(r.get("answer", "")).lower() + ), + ) + assert (working / "data/log.txt").read_text(encoding="utf-8") == "line1\nline2\n" + print("✓ test_append_non_md_warns passed") + + _run(run()) + + +def test_read_non_utf8_encoding(): + """A GBK-encoded legacy file (e.g. CN-Windows CSV) is decoded via the GBK fallback.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + target = working / "data.csv" + target.parent.mkdir(parents=True, exist_ok=True) + text = "姓名,职业\n你好世界,工程师\n" + target.write_bytes(text.encode("gbk")) + async with mock_reme_server() as (host, port): + await call_and_check( + "read", + host=host, + port=port, + path="data.csv", + validator=lambda r: (r.get("success") is True and "你好世界" in str(r.get("answer", ""))), + ) + print("✓ test_read_non_utf8_encoding passed") + + _run(run()) + + +def test_append_preserves_gbk_encoding(): + """Appending to a GBK file re-encodes new content in GBK (no UTF-8 corruption).""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + target = working / "data.csv" + target.parent.mkdir(parents=True, exist_ok=True) + existing = ("姓名,年龄\n张三,30\n" * 20).encode("gbk") + target.write_bytes(existing) + async with mock_reme_server() as (host, port): + await call_and_check( + "append", + host=host, + port=port, + path="data.csv", + content="李四,25\n", + validator=lambda r: r.get("success") is True, + ) + # File must round-trip as GBK; UTF-8 decoding would fail or yield mojibake. + raw = target.read_bytes() + assert raw.endswith("李四,25\n".encode("gbk")), raw[-20:] + decoded = raw.decode("gbk") + assert "张三" in decoded and "李四" in decoded + print("✓ test_append_preserves_gbk_encoding passed") + + _run(run()) + + +def test_edit_preserves_gbk_encoding(): + """Editing a GBK file keeps the file encoded in GBK after the rewrite.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + target = working / "notes.csv" + target.parent.mkdir(parents=True, exist_ok=True) + text = "原始内容,占位\n" * 20 + target.write_bytes(text.encode("gbk")) + async with mock_reme_server() as (host, port): + await call_and_check( + "edit", + host=host, + port=port, + path="notes.csv", + old="原始内容", + new="替换后", + validator=lambda r: r.get("success") is True, + ) + raw = target.read_bytes() + # File still decodes as GBK (would raise if we'd silently converted to UTF-8). + decoded = raw.decode("gbk") + assert "替换后" in decoded and "原始内容" not in decoded + print("✓ test_edit_preserves_gbk_encoding passed") + + _run(run()) + + +def test_read_utf8_bom(): + """Reading a UTF-8 file with BOM strips the BOM transparently.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + target = working / "bom.txt" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(b"\xef\xbb\xbfhello world\n") + async with mock_reme_server() as (host, port): + await call_and_check( + "read", + host=host, + port=port, + path="bom.txt", + validator=lambda r: ( + r.get("success") is True + and "hello world" in str(r.get("answer", "")) + and "" not in str(r.get("answer", "")) + ), + ) + print("✓ test_read_utf8_bom passed") + + _run(run()) + + +# -- aggregate: reuse one server for all read cases ---------------------- + + +def test_all_read_cases_one_server(): + """Run multiple read scenarios against a single shared server for efficiency.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "Templates/Recipe.md", "# Recipe\nbody\n") + _seed_md(working, "Notes.md", "L1\nL2\nL3\n") + async with mock_reme_server() as (host, port): + await call_and_check( + "read", + host=host, + port=port, + path="Templates/Recipe.md", + validator=lambda r: r.get("success") is True and "# Recipe" in r["answer"], + ) + await call_and_check( + "read", + host=host, + port=port, + path="Notes", + validator=lambda r: r.get("success") is True and "L1" in r["answer"], + ) + await call_and_check( + "read", + host=host, + port=port, + path="Notes.md", + start_line=2, + end_line=2, + validator=lambda r: r.get("success") is True and r["answer"].strip() == "L2", + ) + print("✓ test_all_read_cases_one_server passed") + + _run(run()) + + +if __name__ == "__main__": + print("\n=== crud step tests (opaque-byte surface) ===") + # stat / list / download / move / delete + test_stat_indexed_file() + test_stat_directory_fallback() + test_list_lists_files() + test_list_respects_limit_and_non_recursive() + test_download_to_explicit_path() + test_download_to_temp_when_dst_path_empty() + test_move_relocates_within_vault() + test_move_refuses_overwrite_without_flag() + test_move_default_retargets_inbound_links() + test_move_opt_out_leaves_links_dangling() + test_delete_removes_file() + test_delete_missing_returns_error() + test_delete_reports_inbound_refs() + test_delete_folder_removes_tree() + test_delete_folder_reports_only_external_inbound() + test_delete_folder_empty_has_no_inbound() + print("\n=== crud_md (read) E2E tests ===") + test_read_relative_path() + test_read_no_suffix_autoappends_md() + test_read_line_range() + test_read_absolute_path_accepted() + test_read_non_md_degraded() + test_read_missing_file() + test_read_start_after_end() + test_read_start_line_exceeds_total() + test_read_truncation() + test_read_empty_path_rejected() + test_all_read_cases_one_server() + print("\n=== crud_md (write/edit/append) E2E tests ===") + test_write_basic_with_frontmatter() + test_write_no_suffix_autoappends_md() + test_write_overwrites_with_notice() + test_write_creates_parent_dirs() + test_write_no_frontmatter_when_all_empty() + test_write_ignores_arbitrary_extra_fields() + test_write_only_description_present() + test_edit_global_replace() + test_edit_old_not_found() + test_edit_missing_file() + test_edit_skips_frontmatter() + test_edit_match_only_in_frontmatter_fails() + test_append_basic() + test_append_concatenates_verbatim() + test_append_auto_creates_missing_file() + test_append_empty_content_on_existing_file_is_noop() + test_append_empty_content_creates_empty_file() + print("\n=== crud_md (non-md degraded mode) E2E tests ===") + test_write_non_md_skips_frontmatter() + test_edit_non_md_full_text() + test_append_non_md_warns() + test_read_non_utf8_encoding() + test_append_preserves_gbk_encoding() + test_edit_preserves_gbk_encoding() + test_read_utf8_bom() + print("\n所有测试通过!") diff --git a/tests4/unittest/test_daily_steps.py b/tests4/unittest/test_daily_steps.py new file mode 100644 index 00000000..915c5c3b --- /dev/null +++ b/tests4/unittest/test_daily_steps.py @@ -0,0 +1,701 @@ +"""Tests for daily-aware steps: daily_resolve / daily_create / daily_list / daily_reindex. + +Sets up a small ``daily/`` tree with mixed dates and exercises note +genesis + list + index-rebuild operations. Body reads / writes are +generic CRUD (covered in test_crud_steps); arbitrary frontmatter +mutation is covered in test_property_steps. + +A daily note is the single file ``daily//.md`` +(no folder, no sibling materials). ``daily_resolve`` ensures the day +folder ``daily//`` exists and returns the vault-relative path +to the note file, reporting whether it already ``exists`` — it +does **not** create the file itself. ``daily_create`` writes the +note stub with minimal ``name`` frontmatter and refreshes the day +index. + +``daily_list`` and ``daily_reindex`` both call ``refresh_day_index`` +(daily_list as a side effect; daily_reindex as its primary act). They +differ in payload shape: daily_list returns the per-note inventory +(read view), daily_reindex returns the write-result fields (write view). + +Note: status / lifecycle / scope / role / source are no longer +core-reserved fields — the reme schema reserves only name / +description (both optional). Opinionated state machines belong +to the plugin layer. +""" + +# pylint: disable=protected-access + +import asyncio +import os +import tempfile +from datetime import date as _date +from pathlib import Path + +import warnings + +from reme4.components.file_store import LocalFileStore +from reme4.steps.daily import ( + resolve as daily_resolve_step, + create as daily_create_step, + list as daily_list_step, + reindex as daily_reindex_step, +) + +warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources") + + +class temp_chdir: + """Context manager: chdir into a path on enter, restore on exit.""" + + def __init__(self, path): + self.path = path + self.old = None + + def __enter__(self): + self.old = os.getcwd() + os.chdir(self.path) + return self + + def __exit__(self, *_): + os.chdir(self.old) + + +def _today() -> str: + return _date.today().isoformat() + + +async def _make_store_with_dailies(entries: list[tuple[str, str, str]]) -> LocalFileStore: + """Seed the vault with daily notes. + + entries: list of (date, slug, body). Each tuple creates + ``daily//.md`` with a minimal ``name``-only + frontmatter — no opinionated status / lifecycle axes. + """ + store = LocalFileStore(store_name="t", embedding_model="") + await store.start() + for day, slug, body in entries: + day_dir = Path.cwd() / "daily" / day + day_dir.mkdir(parents=True, exist_ok=True) + text = f"---\nname: {slug}\n---\n{body}\n" + (day_dir / f"{slug}.md").write_text(text, encoding="utf-8") + return store + + +def _metadata(step) -> dict: + return step.context.response.metadata + + +async def _seed_note(date: str, slug: str, name: str = "", description: str = "") -> None: + """Write ``daily//.md`` with optional frontmatter.""" + day_dir = Path.cwd() / "daily" / date + day_dir.mkdir(parents=True, exist_ok=True) + fm_lines = [f"name: {name or slug}"] + if description: + fm_lines.append(f"description: {description}") + text = "---\n" + "\n".join(fm_lines) + "\n---\nbody\n" + (day_dir / f"{slug}.md").write_text(text, encoding="utf-8") + + +# -- daily_list_step ---------------------------------------------------------- + + +def test_daily_list_default_date_is_today(): + """No ``date`` arg ⇒ falls back to today; only today's notes returned.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store_with_dailies( + [ + (_today(), "today-a", "today a"), + (_today(), "today-b", "today b"), + ("2026-05-17", "yesterday", "y"), + ], + ) + step = daily_list_step.DailyListStep(file_store=store) + await step() + payload = _metadata(step) + assert payload["date"] == _today() + paths = sorted(n["path"] for n in payload["notes"]) + assert paths == [ + f"daily/{_today()}/today-a.md", + f"daily/{_today()}/today-b.md", + ] + await store.close() + print("✓ test_daily_list_default_date_is_today passed") + + asyncio.run(run()) + + +def test_daily_list_filters_by_date(): + """Explicit ``date`` scopes to that day's folder.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store_with_dailies( + [ + ("2026-05-18", "a", "a"), + ("2026-05-17", "b", "b"), + ], + ) + step = daily_list_step.DailyListStep(file_store=store) + await step(date="2026-05-18") + payload = _metadata(step) + assert payload["date"] == "2026-05-18" + paths = [n["path"] for n in payload["notes"]] + assert paths == ["daily/2026-05-18/a.md"] + await store.close() + print("✓ test_daily_list_filters_by_date passed") + + asyncio.run(run()) + + +def test_daily_list_returns_path_name_description(): + """Each note row exposes path / name / description (and nothing else).""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = LocalFileStore(store_name="t", embedding_model="") + await store.start() + await _seed_note( + "2026-05-18", + "alpha", + name="Alpha Project", + description="JWT auth migration", + ) + step = daily_list_step.DailyListStep(file_store=store) + await step(date="2026-05-18") + payload = _metadata(step) + assert payload["notes"] == [ + { + "path": "daily/2026-05-18/alpha.md", + "name": "Alpha Project", + "description": "JWT auth migration", + }, + ] + await store.close() + print("✓ test_daily_list_returns_path_name_description passed") + + asyncio.run(run()) + + +def test_daily_list_ignores_subdirectories(): + """Subdirectories under the day folder are skipped — only direct .md files count.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store_with_dailies( + [ + ("2026-05-18", "main", "main body"), + ], + ) + # Stray subdir (e.g. left over from an old folder-shaped layout) + # should not be picked up as a note. + stray = Path(tmp) / "daily" / "2026-05-18" / "old-folder" + stray.mkdir(parents=True, exist_ok=True) + (stray / "old-folder.md").write_text( + "---\nname: old\n---\nstale\n", + encoding="utf-8", + ) + + step = daily_list_step.DailyListStep(file_store=store) + await step(date="2026-05-18") + payload = _metadata(step) + paths = [n["path"] for n in payload["notes"]] + assert paths == ["daily/2026-05-18/main.md"] + await store.close() + print("✓ test_daily_list_ignores_subdirectories passed") + + asyncio.run(run()) + + +def test_daily_list_empty_when_no_daily_dir(): + """No daily/ folder ⇒ empty notes list, no crash.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = LocalFileStore(store_name="t", embedding_model="") + await store.start() + step = daily_list_step.DailyListStep(file_store=store) + await step(date="2026-05-18") + payload = _metadata(step) + assert payload == {"date": "2026-05-18", "notes": []} + await store.close() + print("✓ test_daily_list_empty_when_no_daily_dir passed") + + asyncio.run(run()) + + +def test_daily_list_triggers_index_refresh_as_side_effect(): + """Calling daily_list also rebuilds daily/.md (the index page).""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store_with_dailies( + [ + ("2026-05-18", "alpha", "a"), + ("2026-05-18", "beta", "b"), + ], + ) + index_path = Path(tmp) / "daily" / "2026-05-18.md" + assert not index_path.exists() + + step = daily_list_step.DailyListStep(file_store=store) + await step(date="2026-05-18") + + assert index_path.is_file() + text = index_path.read_text(encoding="utf-8") + assert "[[daily/2026-05-18/alpha.md]]" in text + assert "[[daily/2026-05-18/beta.md]]" in text + await store.close() + print("✓ test_daily_list_triggers_index_refresh_as_side_effect passed") + + asyncio.run(run()) + + +def test_daily_list_response_excludes_index_page_fields(): + """daily_list is the read view — no `path` / `created` fields leak through.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store_with_dailies( + [ + ("2026-05-18", "alpha", "a"), + ], + ) + step = daily_list_step.DailyListStep(file_store=store) + await step(date="2026-05-18") + payload = _metadata(step) + assert set(payload.keys()) == {"date", "notes"} + await store.close() + print("✓ test_daily_list_response_excludes_index_page_fields passed") + + asyncio.run(run()) + + +# -- daily_resolve_step ------------------------------------------------------- + + +def test_daily_resolve_ensures_day_folder_and_reports_missing_file(): + """daily_resolve on a fresh name creates the day folder, leaves the note + file unwritten, and reports ``exists=False``.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store_with_dailies([]) + step = daily_resolve_step.DailyResolveStep(file_store=store) + await step(name="kickoff") + payload = _metadata(step) + + assert payload["exists"] is False + assert payload["name"] == "kickoff" + assert payload["date"] == _today() + assert payload["path"] == f"daily/{_today()}/kickoff.md" + assert "message" not in payload + + day_dir = Path(tmp) / "daily" / _today() + assert day_dir.is_dir() + # The note file itself is NOT created by resolve. + assert not (day_dir / "kickoff.md").exists() + # No index page either. + assert not (Path(tmp) / "daily" / f"{_today()}.md").exists() + await store.close() + print("✓ test_daily_resolve_ensures_day_folder_and_reports_missing_file passed") + + asyncio.run(run()) + + +def test_daily_resolve_idempotent_when_file_exists(): + """Existing note file ⇒ ``exists=True`` + message; file contents untouched.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store_with_dailies( + [(_today(), "ongoing", "morning thoughts")], + ) + file_path = Path(tmp) / "daily" / _today() / "ongoing.md" + before = file_path.read_text(encoding="utf-8") + + step = daily_resolve_step.DailyResolveStep(file_store=store) + await step(name="ongoing") + payload = _metadata(step) + + assert payload["exists"] is True + assert payload["name"] == "ongoing" + assert payload["path"] == f"daily/{_today()}/ongoing.md" + assert "already exists" in payload["message"] + + # Contents unchanged. + assert file_path.read_text(encoding="utf-8") == before + await store.close() + print("✓ test_daily_resolve_idempotent_when_file_exists passed") + + asyncio.run(run()) + + +def test_daily_resolve_rejects_empty_name(): + """Empty name ⇒ error payload, success=False, no day folder created.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store_with_dailies([]) + step = daily_resolve_step.DailyResolveStep(file_store=store) + await step(name="") + payload = _metadata(step) + + assert "error" in payload + assert "required" in payload["error"] + assert step.context.response.success is False + assert not (Path(tmp) / "daily" / _today()).exists() + await store.close() + print("✓ test_daily_resolve_rejects_empty_name passed") + + asyncio.run(run()) + + +def test_daily_resolve_rejects_windows_invalid_chars(): + """Windows-reserved characters in name ⇒ error, no day folder created.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store_with_dailies([]) + step = daily_resolve_step.DailyResolveStep(file_store=store) + for bad in ( + "foo/bar", + "foo:bar", + "foo*bar", + "foo?bar", + "foo|bar", + "foobar", + 'foo"bar', + "foo\\bar", + ): + await step(name=bad) + payload = _metadata(step) + assert "error" in payload, f"expected error for {bad!r}, got {payload!r}" + assert "invalid characters" in payload["error"] + assert step.context.response.success is False + await store.close() + print("✓ test_daily_resolve_rejects_windows_invalid_chars passed") + + asyncio.run(run()) + + +def test_daily_resolve_rejects_windows_reserved_names(): + """Windows device-name stems (CON / PRN / AUX / NUL / COM1-9 / LPT1-9) are rejected.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store_with_dailies([]) + step = daily_resolve_step.DailyResolveStep(file_store=store) + for bad in ("CON", "prn", "Aux", "NUL", "COM1", "lpt9", "CON.notes", "com5.txt"): + await step(name=bad) + payload = _metadata(step) + assert "error" in payload, f"expected error for {bad!r}, got {payload!r}" + assert "reserved" in payload["error"] + await store.close() + print("✓ test_daily_resolve_rejects_windows_reserved_names passed") + + asyncio.run(run()) + + +def test_daily_resolve_rejects_trailing_dot_or_whitespace(): + """Trailing '.' / leading-or-trailing whitespace are rejected.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store_with_dailies([]) + step = daily_resolve_step.DailyResolveStep(file_store=store) + for bad in ("foo.", "foo ", " foo", " bar "): + await step(name=bad) + payload = _metadata(step) + assert "error" in payload, f"expected error for {bad!r}, got {payload!r}" + await store.close() + print("✓ test_daily_resolve_rejects_trailing_dot_or_whitespace passed") + + asyncio.run(run()) + + +# -- daily_create_step -------------------------------------------------------- + + +def test_daily_create_writes_file_and_refreshes_index(): + """Fresh slug ⇒ note file created with `name` frontmatter + index refreshed.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store_with_dailies([]) + step = daily_create_step.DailyCreateStep(file_store=store) + await step(slug="kickoff", date="2026-05-18", name="Kickoff Task", body="hello body") + payload = _metadata(step) + + assert payload["created"] is True + assert payload["date"] == "2026-05-18" + assert payload["slug"] == "kickoff" + assert payload["path"] == "daily/2026-05-18/kickoff.md" + + note = Path(tmp) / "daily" / "2026-05-18" / "kickoff.md" + assert note.is_file() + text = note.read_text(encoding="utf-8") + assert "name: Kickoff Task" in text + assert "hello body" in text + + # Index page refreshed. + index = Path(tmp) / "daily" / "2026-05-18.md" + assert index.is_file() + assert "[[daily/2026-05-18/kickoff.md]]" in index.read_text(encoding="utf-8") + await store.close() + print("✓ test_daily_create_writes_file_and_refreshes_index passed") + + asyncio.run(run()) + + +def test_daily_create_is_idempotent(): + """Existing note ⇒ `created=False`, file untouched, index still refreshes.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store_with_dailies( + [("2026-05-18", "ongoing", "old body")], + ) + file_path = Path(tmp) / "daily" / "2026-05-18" / "ongoing.md" + before = file_path.read_text(encoding="utf-8") + + step = daily_create_step.DailyCreateStep(file_store=store) + await step(slug="ongoing", date="2026-05-18", body="ignored new body") + payload = _metadata(step) + + assert payload["created"] is False + assert payload["path"] == "daily/2026-05-18/ongoing.md" + # File contents unchanged. + assert file_path.read_text(encoding="utf-8") == before + # But the index was still rebuilt. + assert payload["index"]["path"] == "daily/2026-05-18.md" + await store.close() + print("✓ test_daily_create_is_idempotent passed") + + asyncio.run(run()) + + +def test_daily_create_name_falls_back_to_slug(): + """Omitted ``name`` arg ⇒ frontmatter ``name`` defaults to slug.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store_with_dailies([]) + step = daily_create_step.DailyCreateStep(file_store=store) + await step(slug="auth-refactor", date="2026-05-18") + + note = Path(tmp) / "daily" / "2026-05-18" / "auth-refactor.md" + assert "name: auth-refactor" in note.read_text(encoding="utf-8") + await store.close() + print("✓ test_daily_create_name_falls_back_to_slug passed") + + asyncio.run(run()) + + +# -- day index: daily/.md ------------------------------------------ + + +def _day_index_text(tmp: str, day: str) -> str: + return (Path(tmp) / "daily" / f"{day}.md").read_text(encoding="utf-8") + + +def test_day_index_lists_each_note(): + """Multiple notes all show up in the index notes block with name.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = LocalFileStore(store_name="t", embedding_model="") + await store.start() + await _seed_note("2026-05-18", "alpha", name="Alpha Project") + await _seed_note("2026-05-18", "beta", name="Beta Project") + + await daily_reindex_step.DailyReindexStep(file_store=store)(date="2026-05-18") + text = _day_index_text(tmp, "2026-05-18") + assert "[[daily/2026-05-18/alpha.md]]" in text + assert "[[daily/2026-05-18/beta.md]]" in text + # Note names show on the indented sub-line. + assert "Alpha Project" in text + assert "Beta Project" in text + await store.close() + print("✓ test_day_index_lists_each_note passed") + + asyncio.run(run()) + + +def test_day_index_includes_note_descriptions(): + """Note ``description`` fields land in the rendered block so the + index reads as a one-glance "what's happening today" summary. + """ + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = LocalFileStore(store_name="t", embedding_model="") + await store.start() + cases = [ + ("alpha", "Alpha Project", "实现 JWT auth 中间件，迁移 session middleware"), + ("beta", "beta", "调研增值税新政对 SaaS 的影响"), # name == slug + ("gamma", "Gamma", ""), # no description + ] + for slug, name, description in cases: + await _seed_note("2026-05-18", slug, name=name, description=description) + + await daily_reindex_step.DailyReindexStep(file_store=store)(date="2026-05-18") + text = _day_index_text(tmp, "2026-05-18") + # name + description rendered together + assert "Alpha Project — 实现 JWT auth 中间件" in text + # name == slug → only description shown (no redundant "beta") + assert "调研增值税新政对 SaaS 的影响" in text + # no description → only name shown, no trailing em-dash + assert " Gamma\n" in text or text.rstrip().endswith("Gamma") + await store.close() + print("✓ test_day_index_includes_note_descriptions passed") + + asyncio.run(run()) + + +def test_day_index_description_is_note_count(): + """The typed ``description`` field carries a one-line note-count digest.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store_with_dailies( + [ + ("2026-05-18", "a", "a body"), + ("2026-05-18", "b", "b body"), + ], + ) + step = daily_reindex_step.DailyReindexStep(file_store=store) + await step(date="2026-05-18") + + text = _day_index_text(tmp, "2026-05-18") + assert "description:" in text + assert "2 篇笔记" in text + await store.close() + print("✓ test_day_index_description_is_note_count passed") + + asyncio.run(run()) + + +def test_day_index_preserves_manual_segment(): + """The ``## 备忘`` (manual) segment is preserved across refreshes.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = LocalFileStore(store_name="t", embedding_model="") + await store.start() + await _seed_note("2026-05-18", "alpha") + reindex = daily_reindex_step.DailyReindexStep(file_store=store) + await reindex(date="2026-05-18") + + # Inject a manual annotation into the index file's body. + index_path = Path(tmp) / "daily" / "2026-05-18.md" + text = index_path.read_text(encoding="utf-8") + patched = text.replace( + "（人工记录区，刷新索引时不会动）", + "MY HAND-WRITTEN NOTE\n这是我手写的备忘，不该被覆盖", + ) + index_path.write_text(patched, encoding="utf-8") + + # Adding a sibling note + refresh — manual segment must survive. + await _seed_note("2026-05-18", "beta") + await reindex(date="2026-05-18") + after = index_path.read_text(encoding="utf-8") + assert "MY HAND-WRITTEN NOTE" in after + assert "这是我手写的备忘" in after + # Auto block was updated with the new note. + assert "[[daily/2026-05-18/beta.md]]" in after + await store.close() + print("✓ test_day_index_preserves_manual_segment passed") + + asyncio.run(run()) + + +# -- daily_reindex_step ----------------------------------------------------- + + +def test_daily_reindex_returns_write_view(): + """daily_reindex returns {date, path, created, notes_count}.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store_with_dailies( + [ + ("2026-05-18", "alpha", "a body"), + ("2026-05-18", "beta", "b body"), + ], + ) + # Index doesn't exist yet. + assert not (Path(tmp) / "daily" / "2026-05-18.md").exists() + + step = daily_reindex_step.DailyReindexStep(file_store=store) + await step(date="2026-05-18") + payload = _metadata(step) + + assert set(payload.keys()) == {"date", "path", "created", "notes_count"} + assert payload["date"] == "2026-05-18" + assert payload["path"] == "daily/2026-05-18.md" + assert payload["created"] is True + assert payload["notes_count"] == 2 + + text = _day_index_text(tmp, "2026-05-18") + assert "[[daily/2026-05-18/alpha.md]]" in text + assert "[[daily/2026-05-18/beta.md]]" in text + await store.close() + print("✓ test_daily_reindex_returns_write_view passed") + + asyncio.run(run()) + + +def test_daily_reindex_created_flag_flips_on_rerun(): + """First call creates the index (created=True); re-run reports created=False.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store_with_dailies( + [("2026-05-18", "alpha", "a")], + ) + step = daily_reindex_step.DailyReindexStep(file_store=store) + + await step(date="2026-05-18") + payload_first = _metadata(step) + assert payload_first["created"] is True + + await step(date="2026-05-18") + payload_second = _metadata(step) + assert payload_second["created"] is False + assert payload_second["notes_count"] == 1 + await store.close() + print("✓ test_daily_reindex_created_flag_flips_on_rerun passed") + + asyncio.run(run()) + + +if __name__ == "__main__": + print("\n=== Daily step tests ===") + test_daily_list_default_date_is_today() + test_daily_list_filters_by_date() + test_daily_list_returns_path_name_description() + test_daily_list_ignores_subdirectories() + test_daily_list_empty_when_no_daily_dir() + test_daily_list_triggers_index_refresh_as_side_effect() + test_daily_list_response_excludes_index_page_fields() + test_daily_resolve_ensures_day_folder_and_reports_missing_file() + test_daily_resolve_idempotent_when_file_exists() + test_daily_resolve_rejects_empty_name() + test_daily_resolve_rejects_windows_invalid_chars() + test_daily_resolve_rejects_windows_reserved_names() + test_daily_resolve_rejects_trailing_dot_or_whitespace() + test_daily_create_writes_file_and_refreshes_index() + test_daily_create_is_idempotent() + test_daily_create_name_falls_back_to_slug() + test_day_index_lists_each_note() + test_day_index_includes_note_descriptions() + test_day_index_description_is_note_count() + test_day_index_preserves_manual_segment() + test_daily_reindex_returns_write_view() + test_daily_reindex_created_flag_flips_on_rerun() + print("\nAll tests passed!") diff --git a/tests4/unittest/test_resource_steps.py b/tests4/unittest/test_resource_steps.py new file mode 100644 index 00000000..aa44b622 --- /dev/null +++ b/tests4/unittest/test_resource_steps.py @@ -0,0 +1,605 @@ +"""Tests for the resource ingest path: ``UploadResourceStep`` + helpers. + +``upload_resource`` is the **passive** ingest entry point — external channels +push assets into ``resource//``, where each call appends a +:class:`FileNode` row to ``meta.json`` (provenance on +``front_matter``) and regenerates the day's ``.md`` view from +the updated meta. These tests exercise that contract end-to-end on a +temp vault, plus the pure ``_assemble_day_md`` helper in isolation. + +The bucket file name is always derived: ``____``. +Duplicates surface as errors (no silent suffixing). +""" + +# pylint: disable=protected-access + +import asyncio +import datetime +import json +import os +import re +import tempfile +import warnings +from pathlib import Path + +from reme4.components.file_store import LocalFileStore +from reme4.schema import FileFrontMatter, FileNode +from reme4.steps.crud import upload_resource as crud_upload +from reme4.steps.crud.upload_resource import _assemble_day_md + +warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources") + + +class temp_chdir: + """Context manager to temporarily chdir into a path and restore on exit.""" + + def __init__(self, path): + self.path = path + self.old = None + + def __enter__(self): + self.old = os.getcwd() + os.chdir(self.path) + return self + + def __exit__(self, *exc): + os.chdir(self.old) + + +async def _make_store() -> LocalFileStore: + """Minimal LocalFileStore (embedding disabled). vault_path resolves to CWD.""" + store = LocalFileStore(store_name="t", embedding_model="") + await store.start() + return store + + +def _metadata(step) -> dict: + return step.context.response.metadata + + +def _meta(tmp: str, date: str) -> list[dict]: + return json.loads((Path(tmp) / "resource" / date / "meta.json").read_text(encoding="utf-8")) + + +def _today() -> str: + return datetime.datetime.now().strftime("%Y-%m-%d") + + +# Matches the canonical derived filename shape. +_NAME_RE = re.compile(r"^([a-z0-9][a-z0-9-]*)__(\d{6})__(.+)$") + + +# -- _assemble_day_md ---------------------------------------------------- + + +def _entry(name: str, **fm_fields) -> FileNode: + """Build a FileNode resource entry: path = resource//, all + provenance fields go onto front_matter (extras allowed).""" + return FileNode( + path=f"resource/2026-05-22/{name}", + st_mtime=0.0, + front_matter=FileFrontMatter(**fm_fields), + ) + + +def test_assemble_day_md_renders_entries(): + """The derived view lists entries with channel / source / time / description.""" + entries = [ + _entry( + "wechat__143000__report.pdf", + description="Q1 report", + channel="wechat", + source="design-group", + received_at="2026-05-22T14:30:00", + ), + _entry("browser__095501__bare.png", channel="browser"), + ] + md = _assemble_day_md(entries, "2026-05-22") + assert "name: 2026-05-22" in md + assert "assets: [wechat__143000__report.pdf, browser__095501__bare.png]" in md + assert ( + "- [[resource/2026-05-22/wechat__143000__report.pdf]] — wechat from `design-group` at 14:30 — Q1 report" in md + ) + assert "- [[resource/2026-05-22/browser__095501__bare.png]] — browser" in md + print("✓ test_assemble_day_md_renders_entries passed") + + +def test_assemble_day_md_empty_bucket(): + """An empty bucket still produces a well-formed frontmatter + header.""" + md = _assemble_day_md([], "2026-05-22") + assert "assets: []" in md + assert "# 2026-05-22 resources" in md + print("✓ test_assemble_day_md_empty_bucket passed") + + +# -- _validate_basename (direct, pure) ----------------------------------- + + +def test_validate_basename_rejects_path_separators(): + """Path-separator basenames are rejected even if the public API can no + longer reach this code path (Path(...).name strips them) — defense in depth.""" + for bad in ("evil/payload.pdf", "..\\winpath.pdf", "../escape.pdf"): + err = crud_upload._validate_basename(bad) + assert "path separators" in err or "reserved" in err, (bad, err) + print("✓ test_validate_basename_rejects_path_separators passed") + + +def test_validate_basename_rejects_dot_segments(): + """`.` and `..` are explicitly reserved.""" + for bad in (".", ".."): + err = crud_upload._validate_basename(bad) + assert "reserved" in err or "start with '.'" in err, (bad, err) + print("✓ test_validate_basename_rejects_dot_segments passed") + + +# -- _validate_channel (direct, pure) ------------------------------------ + + +def test_validate_channel_accepts_safe_identifiers(): + """Lowercase letters / digits / dashes, starting alnum — all accepted.""" + for ok in ("wechat", "email", "api", "browser", "slack-1", "ch1"): + assert crud_upload._validate_channel(ok) == "", ok + print("✓ test_validate_channel_accepts_safe_identifiers passed") + + +def test_validate_channel_rejects_unsafe_identifiers(): + """Uppercase, underscores, leading dash, empty, special chars — rejected.""" + for bad in ("", "WeChat", "we_chat", "-leading", "we chat", "we/chat", "我"): + err = crud_upload._validate_channel(bad) + assert err, bad + print("✓ test_validate_channel_rejects_unsafe_identifiers passed") + + +# -- UploadResourceStep end-to-end -------------------------------------- + + +def test_upload_first_call_creates_bucket(): + """First upload creates resource//, copies the asset under the derived name.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store() + src = Path(tmp) / "incoming.pdf" + src.write_bytes(b"%PDF-fake") + + step = crud_upload.UploadResourceStep(file_store=store) + await step( + path=str(src), + channel="wechat", + description="Q1 report", + metadata={"source": "design-group"}, + ) + payload = _metadata(step) + assert "error" not in payload, payload + date = payload["date"] + assert date == _today() + + m = _NAME_RE.match(payload["name"]) + assert m is not None, payload["name"] + assert m.group(1) == "wechat" + assert m.group(3) == "incoming.pdf" + assert payload["path"] == f"resource/{date}/{payload['name']}" + + bucket = Path(tmp) / "resource" / date + assert (bucket / payload["name"]).read_bytes() == b"%PDF-fake" + + meta = _meta(tmp, date) + assert len(meta) == 1 + assert Path(meta[0]["path"]).name == payload["name"] + fm = meta[0]["front_matter"] + assert fm["channel"] == "wechat" + assert fm["source"] == "design-group" + assert fm["description"] == "Q1 report" + + day_md = (bucket / f"{date}.md").read_text(encoding="utf-8") + assert f"name: {date}" in day_md + assert payload["name"] in day_md + await store.close() + print("✓ test_upload_first_call_creates_bucket passed") + + asyncio.run(run()) + + +def test_upload_metadata_optional(): + """metadata is optional — minimal call is just path + channel + description.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store() + src = Path(tmp) / "small.txt" + src.write_text("x") + step = crud_upload.UploadResourceStep(file_store=store) + await step(path=str(src), channel="api", description="minimal") + payload = _metadata(step) + assert "error" not in payload, payload + row = _meta(tmp, payload["date"])[0] + fm = row["front_matter"] + assert fm["channel"] == "api" + assert fm.get("source", "") == "" + await store.close() + print("✓ test_upload_metadata_optional passed") + + asyncio.run(run()) + + +def test_upload_appends_to_existing_meta(): + """Subsequent uploads append to meta.json and regenerate .md.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store() + names = [] + for i, suffix in enumerate(("first", "second"), start=1): + src = Path(tmp) / f"{suffix}.txt" + src.write_text(f"payload-{i}") + step = crud_upload.UploadResourceStep(file_store=store) + await step(path=str(src), channel="email", description=f"item {i}") + payload = _metadata(step) + assert "error" not in payload, payload + names.append(payload["name"]) + + date = _today() + meta = _meta(tmp, date) + assert [Path(row["path"]).name for row in meta] == names + + day_md = (Path(tmp) / "resource" / date / f"{date}.md").read_text(encoding="utf-8") + assert f"assets: [{', '.join(names)}]" in day_md + for name in names: + assert f"[[resource/{date}/{name}]]" in day_md + await store.close() + print("✓ test_upload_appends_to_existing_meta passed") + + asyncio.run(run()) + + +def test_upload_errors_on_duplicate_same_second(monkeypatch): + """Two uploads of the same (channel, second, basename) → second one errors out.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store() + # Pin time so both calls land in the same HHMMSS slot deterministically. + fixed = datetime.datetime(2026, 5, 22, 15, 30, 22) + + class _FrozenDT(datetime.datetime): + @classmethod + def now(cls, tz=None): # pylint: disable=unused-argument + return fixed + + monkeypatch.setattr(crud_upload.datetime, "datetime", _FrozenDT) + + for i, body in enumerate((b"alpha", b"beta")): + src = Path(tmp) / "incoming.pdf" + src.write_bytes(body) + step = crud_upload.UploadResourceStep(file_store=store) + await step(path=str(src), channel="wechat", description="dup test") + payload = _metadata(step) + if i == 0: + assert "error" not in payload, payload + else: + assert "duplicate" in payload.get("error", "").lower(), payload + + bucket = Path(tmp) / "resource" / "2026-05-22" + # Only the first upload's file should be on disk. + payloads = [p for p in bucket.iterdir() if p.is_file() and p.name.endswith(".pdf")] + assert len(payloads) == 1 + assert payloads[0].read_bytes() == b"alpha" + meta = _meta(tmp, "2026-05-22") + assert len(meta) == 1 + await store.close() + print("✓ test_upload_errors_on_duplicate_same_second passed") + + asyncio.run(run()) + + +def test_upload_errors_on_duplicate_against_on_disk_stray(monkeypatch): + """A stray file on disk (no meta row) still counts as a collision → error.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store() + fixed = datetime.datetime(2026, 5, 22, 15, 30, 22) + + class _FrozenDT(datetime.datetime): + @classmethod + def now(cls, tz=None): # pylint: disable=unused-argument + return fixed + + monkeypatch.setattr(crud_upload.datetime, "datetime", _FrozenDT) + + bucket = Path(tmp) / "resource" / "2026-05-22" + bucket.mkdir(parents=True) + stray = bucket / "api__153022__report.pdf" + stray.write_bytes(b"orphan") + + src = Path(tmp) / "report.pdf" + src.write_bytes(b"fresh") + step = crud_upload.UploadResourceStep(file_store=store) + await step(path=str(src), channel="api", description="fresh copy") + payload = _metadata(step) + assert "duplicate" in payload.get("error", "").lower(), payload + # Stray untouched. + assert stray.read_bytes() == b"orphan" + await store.close() + print("✓ test_upload_errors_on_duplicate_against_on_disk_stray passed") + + asyncio.run(run()) + + +def test_upload_rejects_missing_source(): + """Missing local file → error, no bucket created.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store() + step = crud_upload.UploadResourceStep(file_store=store) + await step( + path=str(Path(tmp) / "ghost.txt"), + channel="email", + description="x", + ) + payload = _metadata(step) + assert "not found" in payload.get("error", "") + assert not (Path(tmp) / "resource").exists() + await store.close() + print("✓ test_upload_rejects_missing_source passed") + + asyncio.run(run()) + + +def test_upload_requires_channel(): + """Missing / blank / malformed channel → error.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store() + src = Path(tmp) / "x.txt" + src.write_text("x") + for bad in (" ", "WeChat", "we_chat"): + step = crud_upload.UploadResourceStep(file_store=store) + await step(path=str(src), channel=bad, description="x") + payload = _metadata(step) + assert "channel" in payload.get("error", ""), (bad, payload) + await store.close() + print("✓ test_upload_requires_channel passed") + + asyncio.run(run()) + + +def test_upload_requires_description(): + """Empty / blank description → error.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store() + src = Path(tmp) / "x.txt" + src.write_text("x") + step = crud_upload.UploadResourceStep(file_store=store) + await step(path=str(src), channel="api", description=" ") + payload = _metadata(step) + assert "description" in payload.get("error", "") + await store.close() + print("✓ test_upload_requires_description passed") + + asyncio.run(run()) + + +def test_upload_rejects_non_dict_metadata(): + """Passing a non-dict in `metadata=` → error.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store() + src = Path(tmp) / "x.txt" + src.write_text("x") + step = crud_upload.UploadResourceStep(file_store=store) + await step( + path=str(src), + channel="api", + description="x", + metadata="source=foo", + ) + payload = _metadata(step) + assert "metadata" in payload.get("error", "") + await store.close() + print("✓ test_upload_rejects_non_dict_metadata passed") + + asyncio.run(run()) + + +def test_upload_rejects_reserved_metadata_keys(): + """Step-managed keys (`name`, `channel`, `received_at`, `description`) + can't be smuggled in via metadata.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store() + src = Path(tmp) / "x.txt" + src.write_text("x") + for bad in ("name", "channel", "received_at", "description"): + step = crud_upload.UploadResourceStep(file_store=store) + await step( + path=str(src), + channel="api", + description="x", + metadata={bad: "evil"}, + ) + payload = _metadata(step) + assert "reserved" in payload.get("error", ""), (bad, payload) + await store.close() + print("✓ test_upload_rejects_reserved_metadata_keys passed") + + asyncio.run(run()) + + +def test_upload_preserves_extra_metadata_keys(): + """Arbitrary keys in `metadata` land on the meta.json row verbatim.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store() + src = Path(tmp) / "x.txt" + src.write_text("x") + step = crud_upload.UploadResourceStep(file_store=store) + await step( + path=str(src), + channel="api", + description="tagged", + metadata={ + "source": "https://example.com", + "tag": "design", + "priority": 3, + }, + ) + payload = _metadata(step) + assert "error" not in payload, payload + row = _meta(tmp, payload["date"])[0] + fm = row["front_matter"] + assert fm["tag"] == "design" + assert fm["priority"] == 3 + assert fm["source"] == "https://example.com" + await store.close() + print("✓ test_upload_preserves_extra_metadata_keys passed") + + asyncio.run(run()) + + +# -- basename-derivation safety ------------------------------------------ + + +def test_upload_rejects_dotfile_source(): + """A source file whose basename starts with '.' → error.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store() + for bad in (".hidden", ".lock", ".env"): + src = Path(tmp) / bad + src.write_text("x") + step = crud_upload.UploadResourceStep(file_store=store) + await step( + path=str(src), + channel="api", + description="x", + ) + payload = _metadata(step) + assert "start with '.'" in payload.get("error", ""), payload + src.unlink() + await store.close() + print("✓ test_upload_rejects_dotfile_source passed") + + asyncio.run(run()) + + +# -- payload-shape sanity ------------------------------------------------ + + +def test_upload_records_received_at_internally(): + """`received_at` is not a caller param but the step stamps it from the + system clock so the day's .md HH:MM column renders.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store() + src = Path(tmp) / "doc.pdf" + src.write_bytes(b"%PDF") + step = crud_upload.UploadResourceStep(file_store=store) + await step(path=str(src), channel="api", description="x") + payload = _metadata(step) + assert "error" not in payload + meta = _meta(tmp, payload["date"]) + assert len(meta) == 1 + stamped = meta[0]["front_matter"]["received_at"] + parsed = datetime.datetime.fromisoformat(stamped) + assert parsed.strftime("%Y-%m-%d") == payload["date"] + # The HHMMSS slot in the name matches the stamped time. + m = _NAME_RE.match(payload["name"]) + assert m is not None + assert m.group(2) == parsed.strftime("%H%M%S") + await store.close() + print("✓ test_upload_records_received_at_internally passed") + + asyncio.run(run()) + + +def test_upload_preserves_description_verbatim_in_meta(): + """meta.json carries the verbatim multi-line description (downstream agents + rely on it for analysis hints); only the day.md bullet flattens for display.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _make_store() + src = Path(tmp) / "doc.pdf" + src.write_bytes(b"%PDF") + multi = "wechat group screenshot\nfrom design-group at 14:30\nlikely a Q1 KPI table — extract numbers" + step = crud_upload.UploadResourceStep(file_store=store) + await step( + path=str(src), + channel="api", + description=multi, + ) + payload = _metadata(step) + assert "error" not in payload, payload + + # meta.json preserves the original — downstream digester sees full hint. + meta = _meta(tmp, payload["date"]) + assert meta[0]["front_matter"]["description"] == multi + + # day.md bullet is flattened (single line, no embedded newlines). + day_md = (Path(tmp) / "resource" / payload["date"] / f"{payload['date']}.md").read_text(encoding="utf-8") + flat = " ".join(multi.split()) + assert flat in day_md + # The bullet line itself must contain the flattened text — and no embedded newline. + bullet_prefix = f"- [[resource/{payload['date']}/{payload['name']}]]" + bullets = [line for line in day_md.splitlines() if line.startswith(bullet_prefix)] + assert len(bullets) == 1, day_md + assert flat in bullets[0] + await store.close() + print("✓ test_upload_preserves_description_verbatim_in_meta passed") + + asyncio.run(run()) + + +def test_assemble_day_md_flattens_multiline_description(): + """Pure helper: a multi-line description on an entry renders as a single + bullet line with newlines collapsed.""" + entries = [ + _entry( + "api__120000__doc.pdf", + description="line one\nline two\n line three", + channel="api", + received_at="2026-05-22T12:00:00", + ), + ] + md = _assemble_day_md(entries, "2026-05-22") + assert "line one line two line three" in md + # The bullet line must contain the flattened text on a single line. + bullets = [line for line in md.splitlines() if line.startswith("- [[resource/2026-05-22/api__120000__doc.pdf]]")] + assert len(bullets) == 1, md + assert "line one line two line three" in bullets[0] + print("✓ test_assemble_day_md_flattens_multiline_description passed") + + +if __name__ == "__main__": + print("\n=== resource step tests ===") + test_assemble_day_md_renders_entries() + test_assemble_day_md_empty_bucket() + test_validate_basename_rejects_path_separators() + test_validate_basename_rejects_dot_segments() + test_validate_channel_accepts_safe_identifiers() + test_validate_channel_rejects_unsafe_identifiers() + test_upload_first_call_creates_bucket() + test_upload_metadata_optional() + test_upload_appends_to_existing_meta() + test_upload_rejects_missing_source() + test_upload_requires_channel() + test_upload_requires_description() + test_upload_rejects_non_dict_metadata() + test_upload_rejects_reserved_metadata_keys() + test_upload_preserves_extra_metadata_keys() + test_upload_rejects_dotfile_source() + test_upload_records_received_at_internally() + test_upload_preserves_description_verbatim_in_meta() + test_assemble_day_md_flattens_multiline_description() + print("\n所有测试通过!") diff --git a/tests4/unittest/test_wikilink_utils.py b/tests4/unittest/test_wikilink_utils.py new file mode 100644 index 00000000..1454afd2 --- /dev/null +++ b/tests4/unittest/test_wikilink_utils.py @@ -0,0 +1,399 @@ +"""Tests for the wikilink helpers in ``reme4.utils.wikilink_handler``. + +Two pure async helpers used by file_move / file_delete: + + * ``retarget_links(src, dst, scope?, dry_run?)`` — rewrite wikilink + targets across the vault, using the file_graph's reverse index to + find inbound sources (no fs scan). + * ``find_inbound(target, scope?)`` — report inbound count without + rewriting. + +Retarget only matches the literal full-path form ``[[topics/x.md]]``; +short ``[[x]]`` and no-ext ``[[topics/x]]`` are left alone by design. +""" + +# pylint: disable=protected-access + +import asyncio +import os +import tempfile +import warnings +from pathlib import Path + +from reme4.components.file_store import LocalFileStore +from reme4.schema import FileNode +from reme4.utils.wikilink_handler import WikilinkHandler + +warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources") + + +class temp_chdir: + """Test helper: chdir to ``path`` on enter, restore previous cwd on exit.""" + + def __init__(self, path): + self.path = path + self.old = None + + def __enter__(self): + self.old = os.getcwd() + os.chdir(self.path) + return self + + def __exit__(self, *exc): + os.chdir(self.old) + + +async def _store_with(files: dict[str, str]) -> LocalFileStore: + """LocalFileStore seeded with (rel → content) files: written to disk + AND registered in the file_graph with wikilinks parsed from the body. + + Without the parsed links the reverse-index lookup yields nothing and + retarget becomes a no-op. + """ + store = LocalFileStore(store_name="t", embedding_model="") + await store.start() + nodes: list[FileNode] = [] + root = Path.cwd() + for rel, content in files.items(): + abs_path = root / rel + abs_path.parent.mkdir(parents=True, exist_ok=True) + abs_path.write_text(content, encoding="utf-8") + nodes.append( + FileNode( + path=rel, + st_mtime=abs_path.stat().st_mtime, + links=WikilinkHandler.extract_links(content, rel), + ), + ) + if nodes: + await store.file_graph.upsert_nodes(nodes) + return store + + +async def _empty_store() -> LocalFileStore: + store = LocalFileStore(store_name="t", embedding_model="") + await store.start() + return store + + +def test_retarget_exact_full_path_match(): + """[[topics/Alice.md]] → [[people/Alice.md]]; non-matching links untouched.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + root = Path(tmp) + store = await _store_with( + { + "note.md": "See [[topics/Alice.md]] and [[topics/Bob.md]].", + "people/Alice.md": "# Alice", + }, + ) + payload = await WikilinkHandler.retarget_links(store, src="topics/Alice.md", dst="people/Alice.md") + assert "error" not in payload + assert payload["links_changed"] == 1 + assert payload["files_touched"] == 1 + body = (root / "note.md").read_text(encoding="utf-8") + assert "[[people/Alice.md]]" in body + assert "[[topics/Alice.md]]" not in body + assert "[[topics/Bob.md]]" in body + await store.close() + print("✓ test_retarget_exact_full_path_match passed") + + asyncio.run(run()) + + +def test_retarget_short_and_no_ext_forms_ignored(): + """Short-form [[Alice]] and no-ext [[topics/Alice]] are NOT matched.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + root = Path(tmp) + body_in = "See [[Alice]] and [[topics/Alice]] but also [[topics/Alice.md]]." + store = await _store_with( + { + "note.md": body_in, + "people/Alice.md": "# Alice", + }, + ) + payload = await WikilinkHandler.retarget_links(store, src="topics/Alice.md", dst="people/Alice.md") + assert payload["links_changed"] == 1 # only the full-path form + body = (root / "note.md").read_text(encoding="utf-8") + assert "[[people/Alice.md]]" in body + assert "[[Alice]]" in body # short form untouched + assert "[[topics/Alice]]" in body # no-ext form untouched + await store.close() + print("✓ test_retarget_short_and_no_ext_forms_ignored passed") + + asyncio.run(run()) + + +def test_retarget_anchor_preserved(): + """`[[topics/Alice.md#intro]]` → `[[people/Alice.md#intro]]` (anchor kept verbatim).""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + root = Path(tmp) + store = await _store_with({"note.md": "Jump to [[topics/Alice.md#intro]] please."}) + payload = await WikilinkHandler.retarget_links(store, src="topics/Alice.md", dst="people/Alice.md") + assert payload["links_changed"] == 1 + body = (root / "note.md").read_text(encoding="utf-8") + assert "[[people/Alice.md#intro]]" in body + await store.close() + print("✓ test_retarget_anchor_preserved passed") + + asyncio.run(run()) + + +def test_retarget_alias_preserved(): + """`[[topics/Alice.md|Display Name]]` → `[[people/Alice.md|Display Name]]`.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + root = Path(tmp) + store = await _store_with({"note.md": "Meet [[topics/Alice.md|Alice the Architect]]."}) + payload = await WikilinkHandler.retarget_links(store, src="topics/Alice.md", dst="people/Alice.md") + assert payload["links_changed"] == 1 + body = (root / "note.md").read_text(encoding="utf-8") + assert "[[people/Alice.md|Alice the Architect]]" in body + await store.close() + print("✓ test_retarget_alias_preserved passed") + + asyncio.run(run()) + + +def test_retarget_anchor_and_alias_together(): + """`[[A.md#h|disp]]` → `[[B.md#h|disp]]` keeps both suffixes.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + root = Path(tmp) + store = await _store_with({"note.md": "See [[topics/Alice.md#bio|her bio]] now."}) + payload = await WikilinkHandler.retarget_links(store, src="topics/Alice.md", dst="people/Alice.md") + assert payload["links_changed"] == 1 + body = (root / "note.md").read_text(encoding="utf-8") + assert "[[people/Alice.md#bio|her bio]]" in body + await store.close() + print("✓ test_retarget_anchor_and_alias_together passed") + + asyncio.run(run()) + + +def test_retarget_image_marker_preserved(): + """`![[topics/diagram.md]]` (embed) keeps its `!` prefix on rewrite.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + root = Path(tmp) + store = await _store_with({"note.md": "Inline embed: ![[topics/diagram.md]] here."}) + payload = await WikilinkHandler.retarget_links(store, src="topics/diagram.md", dst="diagrams/diagram.md") + assert payload["links_changed"] == 1 + body = (root / "note.md").read_text(encoding="utf-8") + assert "![[diagrams/diagram.md]]" in body + await store.close() + print("✓ test_retarget_image_marker_preserved passed") + + asyncio.run(run()) + + +def test_retarget_dataview_predicate_preserved(): + """Line-level + inline-bracketed Dataview predicates pass through outside ``[[..]]``.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + root = Path(tmp) + body_in = ( + "colleague:: [[topics/Alice.md]]\n" + "She is the [负责:: [[topics/Alice.md]]] for the migration.\n" + ) + store = await _store_with({"note.md": body_in}) + payload = await WikilinkHandler.retarget_links(store, src="topics/Alice.md", dst="people/Alice.md") + assert payload["links_changed"] == 2 + body = (root / "note.md").read_text(encoding="utf-8") + assert "colleague:: [[people/Alice.md]]" in body + assert "[负责:: [[people/Alice.md]]]" in body + await store.close() + print("✓ test_retarget_dataview_predicate_preserved passed") + + asyncio.run(run()) + + +def test_retarget_multiple_files_aggregate_counts(): + """links_changed sums across files; by_file lists per-file counts.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _store_with( + { + "a.md": "[[topics/Alice.md]] then [[topics/Alice.md]]", + "sub/b.md": "[[topics/Alice.md]] and [[topics/Bob.md]]", + }, + ) + payload = await WikilinkHandler.retarget_links(store, src="topics/Alice.md", dst="people/Alice.md") + assert payload["links_changed"] == 3 + assert payload["files_touched"] == 2 + by_file = {row["path"]: row["count"] for row in payload["by_file"]} + assert by_file == {"a.md": 2, "sub/b.md": 1} + await store.close() + print("✓ test_retarget_multiple_files_aggregate_counts passed") + + asyncio.run(run()) + + +def test_retarget_dry_run_does_not_write(): + """dry_run=True reports counts but leaves files on disk unchanged.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + root = Path(tmp) + original = "See [[topics/Alice.md]]." + store = await _store_with({"note.md": original}) + payload = await WikilinkHandler.retarget_links( + store, + src="topics/Alice.md", + dst="people/Alice.md", + dry_run=True, + ) + assert payload["dry_run"] is True + assert payload["links_changed"] == 1 + assert payload["files_touched"] == 1 + assert (root / "note.md").read_text(encoding="utf-8") == original + await store.close() + print("✓ test_retarget_dry_run_does_not_write passed") + + asyncio.run(run()) + + +def test_retarget_scope_limits_sweep(): + """Files outside `scope` are not visited even if they contain matches.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + root = Path(tmp) + store = await _store_with( + { + "in_scope/note.md": "See [[topics/Alice.md]].", + "outside/note.md": "Also [[topics/Alice.md]].", + }, + ) + payload = await WikilinkHandler.retarget_links( + store, + src="topics/Alice.md", + dst="people/Alice.md", + scope="in_scope", + ) + assert payload["links_changed"] == 1 + assert payload["files_touched"] == 1 + in_body = (root / "in_scope/note.md").read_text(encoding="utf-8") + out_body = (root / "outside/note.md").read_text(encoding="utf-8") + assert "[[people/Alice.md]]" in in_body + assert "[[topics/Alice.md]]" in out_body # untouched + await store.close() + print("✓ test_retarget_scope_limits_sweep passed") + + asyncio.run(run()) + + +def test_retarget_src_eq_dst_is_noop(): + """src == dst: no scan needed, returns zero counts.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _store_with({"note.md": "See [[topics/Alice.md]]."}) + payload = await WikilinkHandler.retarget_links(store, src="topics/Alice.md", dst="topics/Alice.md") + assert payload["links_changed"] == 0 + assert payload["files_touched"] == 0 + await store.close() + print("✓ test_retarget_src_eq_dst_is_noop passed") + + asyncio.run(run()) + + +def test_retarget_empty_src_or_dst_errors(): + """Empty src or dst produces an error payload.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _empty_store() + for kwargs in ({"src": "", "dst": "people/Alice.md"}, {"src": "topics/Alice.md", "dst": ""}): + payload = await WikilinkHandler.retarget_links(store, **kwargs) + assert "error" in payload, f"expected error for {kwargs}" + await store.close() + print("✓ test_retarget_empty_src_or_dst_errors passed") + + asyncio.run(run()) + + +def test_retarget_dst_with_forbidden_chars_errors(): + """``dst`` containing ``[ ] # |`` or newline lands as an error payload.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _empty_store() + for bad in ["people/Alice#anchor", "people/Alice|alias", "p/[A].md", "p/A]]"]: + payload = await WikilinkHandler.retarget_links(store, src="topics/Alice.md", dst=bad) + assert "error" in payload, f"expected error for dst={bad!r}, got {payload}" + await store.close() + print("✓ test_retarget_dst_with_forbidden_chars_errors passed") + + asyncio.run(run()) + + +def test_retarget_absolute_path_rejected(): + """Absolute paths in src or dst are rejected (must be relative to the vault).""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _empty_store() + for kwargs in ( + {"src": "/abs/old.md", "dst": "people/Alice.md"}, + {"src": "topics/Alice.md", "dst": "/abs/new.md"}, + ): + payload = await WikilinkHandler.retarget_links(store, **kwargs) + assert "error" in payload, f"expected error for {kwargs}" + await store.close() + print("✓ test_retarget_absolute_path_rejected passed") + + asyncio.run(run()) + + +def test_find_inbound_counts_references(): + """find_inbound reports per-file counts and excludes self-references.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = await _store_with( + { + "a.md": "[[topics/Alice.md]] and [[topics/Alice.md]] again", + "b.md": "see [[topics/Alice.md]]", + "topics/Alice.md": "self-ref [[topics/Alice.md]] should not count", + }, + ) + payload = await WikilinkHandler.find_inbound(store, target="topics/Alice.md") + assert payload["files_touched"] == 2 + assert payload["links_total"] == 3 + by_file = {row["path"]: row["count"] for row in payload["by_file"]} + assert by_file == {"a.md": 2, "b.md": 1} + await store.close() + print("✓ test_find_inbound_counts_references passed") + + asyncio.run(run()) + + +if __name__ == "__main__": + print("\n=== wikilink helper tests ===") + test_retarget_exact_full_path_match() + test_retarget_short_and_no_ext_forms_ignored() + test_retarget_anchor_preserved() + test_retarget_alias_preserved() + test_retarget_anchor_and_alias_together() + test_retarget_image_marker_preserved() + test_retarget_dataview_predicate_preserved() + test_retarget_multiple_files_aggregate_counts() + test_retarget_dry_run_does_not_write() + test_retarget_scope_limits_sweep() + test_retarget_src_eq_dst_is_noop() + test_retarget_empty_src_or_dst_errors() + test_retarget_dst_with_forbidden_chars_errors() + test_retarget_absolute_path_rejected() + test_find_inbound_counts_references() + print("\n所有测试通过!")