From 5677606875f8a8244e0cb94662afb28d628f8a62 Mon Sep 17 00:00:00 2001 From: imrewce Date: Mon, 18 May 2026 18:30:32 +0800 Subject: [PATCH] fix(step): markdown read step fixing pr comments --- docs4/reme_design.md | 6 + reme4/config/default.yaml | 10 +- reme4/steps/__init__.py | 2 + reme4/steps/base_step.py | 71 ++++- reme4/steps/common/__init__.py | 2 - reme4/steps/common/_file_io.py | 159 ---------- reme4/steps/crud_md/__init__.py | 7 + reme4/steps/crud_md/_file_io.py | 113 ++++++++ reme4/steps/{common => crud_md}/read.py | 95 +++--- tests4/unittest/test_crud_md_steps.py | 371 ++++++++++++++++++++++++ 10 files changed, 631 insertions(+), 205 deletions(-) delete mode 100644 reme4/steps/common/_file_io.py create mode 100644 reme4/steps/crud_md/__init__.py create mode 100644 reme4/steps/crud_md/_file_io.py rename reme4/steps/{common => crud_md}/read.py (57%) create mode 100644 tests4/unittest/test_crud_md_steps.py diff --git a/docs4/reme_design.md b/docs4/reme_design.md index 15e7ee28..06c024ad 100644 --- a/docs4/reme_design.md +++ b/docs4/reme_design.md @@ -42,6 +42,7 @@ reme4 version | 🔎 search | 🔍 `search` (`search_step`) | `call_server("search", query=…, …)` | 📥 `query:str` ⭐ ｜ 🎚️ `limit:int=5`(>0) ｜ 🎚️ `min_score:float=0.0` ｜ ⚖️ `vector_weight:float=0.7` ∈[0,1]（keyword 权 = 1-vw）｜ 🔀 `candidate_multiplier:float=3.0`（candidates = min(200, limit×mult)）｜ 🔗 `expand_links:bool=True` ｜ 🔢 `max_links_per_direction:int=10` ｜ 🎚️ `search_filter:dict={}` ｜ 📤 `answer` 每命中一行 `path:start-end [score=… vector=… keyword=…] text` + 缩进的 `→ outlinks (n)` / `← inlinks (n)` + `via predicate=… anchor=#…` ｜ 📊 `metadata.results` / `metadata.link_expansion` / `metadata.counts={vector,keyword,returned,hybrid}` ｜ 🛠️ 并行 `vector_search` + `keyword_search` → RRF 融合（K=60，按 chunk.id 合并）→ `min_score` 过滤 → `limit` 截断 → 邻居 meta 注入 | | 🧪 demo | 🪄 `demo_echo` (`demo_echo_step1` + `step2`) | `call_server("demo_echo", query=…, min_score=…)` | 📥 `query:str=""` ｜ 🎚️ `min_score:float=0.5` ｜ 🛠️ step1：`processed_query = query.strip().lower()`，`adjusted_min_score = min_score * 0.9`，写回 context ｜ 📤 step2：`answer = "echo: {processed_query} (min_score={adjusted_min_score})"` ｜ 📊 `metadata = {step, query, min_score, processed_query, adjusted_min_score}` | | 🌊 demo | 🌊 `stream_demo` (`stream_demo_step1` + `step2`) | `call_server("stream_demo", query=…, repeat=…, interval=…)` | 📥 `query:str=""` ｜ 🎚️ `repeat:int=10` ｜ 🎚️ `interval:float=0.1`（秒/字符）｜ 🛠️ step1：`stream_text = query * repeat` 写回 context ｜ 📤 step2：按字符 `add_stream_string(ch, ChunkEnum.CONTENT)` 流式输出，`asyncio.sleep(interval)` 节流 | +| 📂 crud_md | 📖 `read` (`read_step`) | `call_server("read", path=…, …)` | 📥 `path:str` ⭐（**相对** `working_dir`，绝对路径会被拒绝；无后缀自动补 `.md`，非 `.md` 后缀拒绝）｜ 🎚️ `start_line:int=null`（1-based, 含端点）｜ 🎚️ `end_line:int=null`（1-based, 含端点）｜ 🎚️ `max_bytes:int=51200`（截断阈值）｜ 📤 `answer = 选中的行内容`，超过 `max_bytes` 时附加 `--- TRUNCATED ---` 续读指引（`start_line=…`）｜ 📊 `metadata.path` / `metadata.total_lines`（出错路径才会附带）｜ 🛠️ 流程：`BaseStep.resolve_path(raw)` → `aiofiles.os.stat` → `read_file_safe`（utf-8-sig BOM 容忍、UnicodeDecodeError fallback `errors=ignore`）→ `split("\n")` 切片 `[s-1:e]` → `truncate_text_output` 按字节截断保行 | 使用示例： @@ -66,6 +67,11 @@ reme4 version reme4 reindex reme4 search query="latency 问题" limit=10 min_score=0.2 vector_weight=0.6 +# 读取 working_dir 下的 markdown（相对路径；无后缀自动补 .md；可按行切片或限制字节） +reme4 read path=Templates/Recipe.md +reme4 read path=Notes start_line=1 end_line=20 +reme4 read path=Big.md max_bytes=4096 + # 通过 MCP backend 调用 reme4 search query="..." backend=mcp ``` diff --git a/reme4/config/default.yaml b/reme4/config/default.yaml index e06cf5c4..e4d43568 100644 --- a/reme4/config/default.yaml +++ b/reme4/config/default.yaml @@ -98,23 +98,21 @@ jobs: - backend: base name: read - description: "read a markdown file (absolute path; relative path resolved under .reme)" + description: "read a markdown file (relative to working_dir, no .md suffix needed)" parameters: type: object properties: path: type: string - description: "absolute or relative path; relative is rooted at the reme working_dir (.reme); markdown only (.md auto-appended when no suffix)" + description: "relative path rooted at the working_dir; markdown only (.md auto-appended when no suffix)" start_line: type: integer description: "first line to read (1-based, inclusive)" + default: null end_line: type: integer description: "last line to read (1-based, inclusive)" - max_bytes: - type: integer - description: "max bytes returned before byte-level truncation" - default: 51200 + default: null required: - path steps: diff --git a/reme4/steps/__init__.py b/reme4/steps/__init__.py index fa25273c..5cd169fe 100644 --- a/reme4/steps/__init__.py +++ b/reme4/steps/__init__.py @@ -1,9 +1,11 @@ """steps""" from . import common +from . import crud_md from .base_step import BaseStep __all__ = [ "common", + "crud_md", "BaseStep", ] diff --git a/reme4/steps/base_step.py b/reme4/steps/base_step.py index a710f457..b55c079a 100644 --- a/reme4/steps/base_step.py +++ b/reme4/steps/base_step.py @@ -2,6 +2,7 @@ import copy from abc import abstractmethod, ABC +from pathlib import Path from typing import TypeVar, TYPE_CHECKING from agentscope.formatter import FormatterBase @@ -79,7 +80,49 @@ class BaseStep(ABC): self.context.apply_mapping(self.output_mapping) return result - def _resolve(self, key: str, base_cls: type[T], comp_enum: ComponentEnum, attr: str | None = None) -> T: + @property + def working_path(self) -> Path: + """Resolved working directory from app context or cwd.""" + if self.app_context is None: + return Path.cwd() + return Path(self.app_context.app_config.working_dir) + + def resolve_path( + self, + raw: str, + *, + require_md: bool = True, + ) -> tuple[Path | None, str | None]: + """Resolve a relative `path=` argument under self.working_path. + + Rules: + - relative path only; joined under ``self.working_path``. + Markdown gate (when ``require_md=True``): + - no suffix → auto-append ``.md``; + - any other non-``.md`` suffix → reject. + Returns ``(abs_path, None)`` on success, or ``(None, error_message)`` on failure. + """ + if not raw or not str(raw).strip(): + return None, "`path` is required" + s = str(raw).strip() + p = Path(s) + if p.is_absolute(): + return None, (f"path {s!r} is absolute; only relative paths under the working_dir are accepted") + target = (self.working_path.resolve() / p).resolve() + if require_md: + if target.suffix == "": + target = target.with_suffix(".md") + elif target.suffix.lower() != ".md": + return None, (f"path {s!r} is not a markdown file; this command only supports .md files") + return target, None + + def _resolve( + self, + key: str, + base_cls: type[T], + comp_enum: ComponentEnum, + attr: str | None = None, + ) -> T: """Return a kwargs-supplied instance, or look one up by name in the app registry.""" # 1. Step init kwargs, 2. Runtime context (run_job kwargs), 3. App registry by name. for source in (self.kwargs, self.context or {}): @@ -100,12 +143,22 @@ class BaseStep(ABC): @property def as_llm_formatter(self) -> FormatterBase: """Return the LLM formatter component.""" - return self._resolve("as_llm_formatter", FormatterBase, ComponentEnum.AS_LLM_FORMATTER, "formatter") + return self._resolve( + "as_llm_formatter", + FormatterBase, + ComponentEnum.AS_LLM_FORMATTER, + "formatter", + ) @property def as_token_counter(self) -> TokenCounterBase: """Return the token counter component.""" - return self._resolve("as_token_counter", TokenCounterBase, ComponentEnum.AS_TOKEN_COUNTER, "token_counter") + return self._resolve( + "as_token_counter", + TokenCounterBase, + ComponentEnum.AS_TOKEN_COUNTER, + "token_counter", + ) @property def file_parser(self) -> BaseFileParser: @@ -120,12 +173,20 @@ class BaseStep(ABC): @property def embedding(self) -> BaseEmbeddingModel: """Return the embedding model component.""" - return self._resolve("embedding", BaseEmbeddingModel, ComponentEnum.EMBEDDING_MODEL) + return self._resolve( + "embedding", + BaseEmbeddingModel, + ComponentEnum.EMBEDDING_MODEL, + ) @property def file_watcher(self) -> BaseFileWatcher: """Return the file watcher component.""" - return self._resolve("file_watcher", BaseFileWatcher, ComponentEnum.FILE_WATCHER) + return self._resolve( + "file_watcher", + BaseFileWatcher, + ComponentEnum.FILE_WATCHER, + ) def prompt_format(self, prompt_name: str, **kwargs) -> str: """Format a named prompt template with the given kwargs.""" diff --git a/reme4/steps/common/__init__.py b/reme4/steps/common/__init__.py index bda17e23..e09c2073 100644 --- a/reme4/steps/common/__init__.py +++ b/reme4/steps/common/__init__.py @@ -3,7 +3,6 @@ from .demo import DemoEchoStep1, DemoEchoStep2 from .health_check import HealthCheckStep from .help import HelpStep -from .read import ReadStep from .reindex import ReindexStep from .search import SearchStep from .stream_demo import StreamDemoStep1, StreamDemoStep2 @@ -14,7 +13,6 @@ __all__ = [ "DemoEchoStep2", "HealthCheckStep", "HelpStep", - "ReadStep", "ReindexStep", "SearchStep", "StreamDemoStep1", diff --git a/reme4/steps/common/_file_io.py b/reme4/steps/common/_file_io.py deleted file mode 100644 index ea8b8d80..00000000 --- a/reme4/steps/common/_file_io.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Shared filesystem helpers for CRUD steps (path resolve, safe read, truncation).""" - -import logging -import re -from pathlib import Path - -import aiofiles -import aiofiles.os - -from ...constants import DEFAULT_MAX_BYTES, MAX_FILE_READ_BYTES, TRUNCATION_NOTICE_MARKER - -logger = logging.getLogger(__name__) - - -def resolve_path( - working_path: Path, - raw: str, - *, - require_md: bool = True, -) -> tuple[Path | None, str | None]: - """Resolve a `path=` argument into an absolute Path (qwenpaw-style). - - Rules: - - absolute path → used as-is (with `~` expansion); - - relative path → joined under ``working_path`` (the reme working_dir, default ``.reme``). - Markdown gate (when ``require_md=True``): - - no suffix (e.g. ``My Note``) → auto-append ``.md``; - - any other non-``.md`` suffix → reject. - Returns ``(abs_path, None)`` on success, or ``(None, error_message)`` on failure. - """ - if not raw or not str(raw).strip(): - return None, "`path` is required" - s = str(raw).strip() - p = Path(s).expanduser() - target = p.resolve() if p.is_absolute() else (working_path / p).resolve() - if require_md: - if target.suffix == "": - target = target.with_suffix(".md") - elif target.suffix.lower() != ".md": - return None, ( - f"path {s!r} is not a markdown file; this command only supports .md files" - ) - return target, None - - -async def read_file_safe(file_path: Path | str, max_bytes: int = MAX_FILE_READ_BYTES) -> str: - """Read file with utf-8-sig (BOM-tolerant), fallback to errors='ignore'.""" - stat = await aiofiles.os.stat(str(file_path)) - read_size = min(stat.st_size, max_bytes) - try: - async with aiofiles.open(str(file_path), "r", encoding="utf-8-sig") as f: - return await f.read(read_size) - except UnicodeDecodeError: - async with aiofiles.open( - str(file_path), "r", encoding="utf-8-sig", errors="ignore", - ) as f: - return await f.read(read_size) - - -def truncate_text_output( - text: str, - *, - start_line: int = 1, - total_lines: int = 0, - max_bytes: int = DEFAULT_MAX_BYTES, - file_path: str | None = None, - encoding: str = "utf-8", -) -> str: - """Truncate text by bytes preserving line integrity; append a continuation notice. - - See qwenpaw `tools/utils.py` for the same semantics. Returns text unchanged when - it fits within max_bytes, when max_bytes <= 0, or when the last line itself - exceeds max_bytes (unhandled edge case). - """ - if not text or max_bytes <= 0: - return text - - try: - if TRUNCATION_NOTICE_MARKER in text: - return _retruncate(text, max_bytes=max_bytes, encoding=encoding) - return _truncate_fresh( - text, - start_line=start_line, - total_lines=total_lines, - max_bytes=max_bytes, - file_path=file_path, - encoding=encoding, - ) - except Exception: - logger.warning("truncate_text_output failed, returning original text", exc_info=True) - return text - - -def _truncate_fresh( - text: str, - *, - start_line: int, - total_lines: int, - max_bytes: int, - file_path: str | None, - encoding: str, -) -> str: - text_bytes = text.encode(encoding) - if len(text_bytes) <= max_bytes: - return text - - truncated = text_bytes[:max_bytes] - result = truncated.decode(encoding, errors="ignore") - newline_count = result.count("\n") - next_line = start_line + max(1, newline_count) - - if next_line <= total_lines: - read_from = next_line - elif start_line < total_lines: - read_from = total_lines - else: - return result - - notice = ( - TRUNCATION_NOTICE_MARKER - + f"\nThe output above was truncated." - f"\nThe full content is saved to the file and contains {total_lines} lines in total." - f"\nThis excerpt starts at line {start_line} and covers the next {max_bytes} bytes." - f"\nIf the current content is not enough, call `read` with file={file_path or ''} " - f"start_line={read_from} to read more." - ) - return result + notice - - -def _retruncate(text: str, *, max_bytes: int, encoding: str) -> str: - parts = text.split(TRUNCATION_NOTICE_MARKER, 1) - original_content = parts[0] - old_notice = parts[1] - - text_bytes = original_content.encode(encoding) - if len(text_bytes) <= max_bytes + 100: - return text - - start_match = re.search(r"starts at line (\d+)", old_notice) - if not start_match: - return text - start_line_parsed = int(start_match.group(1)) - - truncated_bytes = text_bytes[:max_bytes] - result = truncated_bytes.decode(encoding, errors="ignore") - newline_count = result.count("\n") - next_line = start_line_parsed + max(1, newline_count) - - if not re.search(r"covers the next \d+ bytes", old_notice): - return text - new_notice = re.sub( - r"covers the next \d+ bytes", f"covers the next {max_bytes} bytes", old_notice, - ) - new_notice = re.sub( - r"start_line=\d+ to read more", f"start_line={next_line} to read more", new_notice, - ) - return result + TRUNCATION_NOTICE_MARKER + new_notice - - diff --git a/reme4/steps/crud_md/__init__.py b/reme4/steps/crud_md/__init__.py new file mode 100644 index 00000000..de9ae9ce --- /dev/null +++ b/reme4/steps/crud_md/__init__.py @@ -0,0 +1,7 @@ +"""CRUD steps for markdown files under the working_dir.""" + +from .read import ReadStep + +__all__ = [ + "ReadStep", +] diff --git a/reme4/steps/crud_md/_file_io.py b/reme4/steps/crud_md/_file_io.py new file mode 100644 index 00000000..44759ff9 --- /dev/null +++ b/reme4/steps/crud_md/_file_io.py @@ -0,0 +1,113 @@ +"""Shared filesystem helpers for CRUD steps (safe read, truncation).""" + +import re + +import aiofiles +import aiofiles.os + +from ...constants import DEFAULT_MAX_BYTES, MAX_FILE_READ_BYTES, TRUNCATION_NOTICE_MARKER +from ...utils import get_logger + +logger = get_logger() + + +async def read_file_safe(file_path, max_bytes: int = MAX_FILE_READ_BYTES) -> str: + """Read file with utf-8-sig (BOM-tolerant), fallback to errors='ignore'.""" + stat = await aiofiles.os.stat(str(file_path)) + read_size = min(stat.st_size, max_bytes) + try: + async with aiofiles.open(str(file_path), "r", encoding="utf-8-sig") as f: + return await f.read(read_size) + except UnicodeDecodeError: + async with aiofiles.open( + str(file_path), + "r", + encoding="utf-8-sig", + errors="ignore", + ) as f: + return await f.read(read_size) + + +def truncate_text_output( + text: str, + *, + start_line: int = 1, + total_lines: int = 0, + max_bytes: int = DEFAULT_MAX_BYTES, + file_path: str | None = None, + encoding: str = "utf-8", +) -> str: + """Truncate text by bytes preserving line integrity; append a continuation notice. + + See qwenpaw `tools/utils.py` for the same semantics. Returns text unchanged when + it fits within max_bytes, when max_bytes <= 0, or when the last line itself + exceeds max_bytes (unhandled edge case). + """ + if not text or max_bytes <= 0: + return text + + try: + if TRUNCATION_NOTICE_MARKER in text: + return _retruncate(text, max_bytes=max_bytes, encoding=encoding) + + text_bytes = text.encode(encoding) + if len(text_bytes) <= max_bytes: + return text + + truncated = text_bytes[:max_bytes] + result = truncated.decode(encoding, errors="ignore") + newline_count = result.count("\n") + next_line = start_line + max(1, newline_count) + + if next_line <= total_lines: + read_from = next_line + elif start_line < total_lines: + read_from = total_lines + else: + return result + + notice = ( + TRUNCATION_NOTICE_MARKER + f"\nThe output above was truncated." + f"\nThe full content is saved to the file and contains {total_lines} lines in total." + f"\nThis excerpt starts at line {start_line} and covers the next {max_bytes} bytes." + f"\nIf the current content is not enough, call `read` with file={file_path or ''} " + f"start_line={read_from} to read more." + ) + return result + notice + except Exception: + logger.warning("truncate_text_output failed, returning original text", exc_info=True) + return text + + +def _retruncate(text: str, *, max_bytes: int, encoding: str) -> str: + parts = text.split(TRUNCATION_NOTICE_MARKER, 1) + original_content = parts[0] + old_notice = parts[1] + + text_bytes = original_content.encode(encoding) + if len(text_bytes) <= max_bytes + 100: + return text + + start_match = re.search(r"starts at line (\d+)", old_notice) + if not start_match: + return text + start_line_parsed = int(start_match.group(1)) + + truncated_bytes = text_bytes[:max_bytes] + result = truncated_bytes.decode(encoding, errors="ignore") + newline_count = result.count("\n") + next_line = start_line_parsed + max(1, newline_count) + + if not re.search(r"covers the next \d+ bytes", old_notice): + return text + new_notice = re.sub( + r"covers the next \d+ bytes", + f"covers the next {max_bytes} bytes", + old_notice, + ) + new_notice = re.sub( + r"start_line=\d+ to read more", + f"start_line={next_line} to read more", + new_notice, + ) + return result + TRUNCATION_NOTICE_MARKER + new_notice diff --git a/reme4/steps/common/read.py b/reme4/steps/crud_md/read.py similarity index 57% rename from reme4/steps/common/read.py rename to reme4/steps/crud_md/read.py index 6db8fb4d..d0657110 100644 --- a/reme4/steps/common/read.py +++ b/reme4/steps/crud_md/read.py @@ -5,7 +5,6 @@ from pathlib import Path from ._file_io import ( DEFAULT_MAX_BYTES, read_file_safe, - resolve_path, truncate_text_output, ) from ..base_step import BaseStep @@ -16,11 +15,6 @@ from ...components import R class ReadStep(BaseStep): """Read a markdown file. Optional `start_line`/`end_line` for ranged reads.""" - def _working_path(self) -> Path: - if self.app_context is not None: - return Path(self.app_context.app_config.working_dir).resolve() - return Path.cwd().resolve() - def _fail(self, message: str, **meta) -> None: assert self.context is not None self.context.response.success = False @@ -28,56 +22,75 @@ class ReadStep(BaseStep): if meta: self.context.response.metadata.update(meta) - async def execute(self): + def _resolve_target_or_fail(self) -> Path | None: assert self.context is not None raw = str(self.context.get("path") or "") - start_line = self.context.get("start_line") - end_line = self.context.get("end_line") - max_bytes = int(self.context.get("max_bytes") or DEFAULT_MAX_BYTES) - - target, err = resolve_path(self._working_path(), raw, require_md=True) + target, err = self.resolve_path(raw, require_md=True) if err: self._fail(err) - return + return None + if not target.exists(): + self._fail(f"file {target} does not exist", path=str(target)) + return None + if not target.is_file(): + self._fail(f"path {target} is not a file", path=str(target)) + return None + return target - for label, value in (("start_line", start_line), ("end_line", end_line)): + def _validate_line_params_or_fail(self) -> bool: + assert self.context is not None + for label in ("start_line", "end_line"): + value = self.context.get(label) if value is None: continue try: int(value) except (TypeError, ValueError): self._fail(f"{label} must be an integer, got {value!r}") - return - - if not target.exists(): - self._fail(f"file {target} does not exist", path=str(target)) - return - if not target.is_file(): - self._fail(f"path {target} is not a file", path=str(target)) - return + return False + return True + async def _load_content_or_fail(self, target: Path) -> str | None: try: - content = await read_file_safe(target) - except Exception as e: - self._fail(f"read failed: {e}", path=str(target)) - return + return await read_file_safe(target) + except Exception as ex: + self._fail(f"read failed: {ex}", path=str(target)) + return None - all_lines = content.split("\n") + def _compute_range_or_fail( + self, + target: Path, + all_lines: list[str], + ) -> tuple[int, int, int] | None: + assert self.context is not None + start_line = self.context.get("start_line") + end_line = self.context.get("end_line") total = len(all_lines) s = max(1, int(start_line) if start_line is not None else 1) e = min(total, int(end_line) if end_line is not None else total) - if s > total: self._fail( f"start_line {s} exceeds file length ({total} lines)", - path=str(target), total_lines=total, + path=str(target), + total_lines=total, ) - return + return None if s > e: self._fail(f"start_line ({s}) > end_line ({e})", path=str(target)) - return + return None + return s, e, total - selected = "\n".join(all_lines[s - 1: e]) + def _emit_response( + self, + target: Path, + all_lines: list[str], + s: int, + e: int, + total: int, + ) -> None: + assert self.context is not None + max_bytes = int(self.context.get("max_bytes") or DEFAULT_MAX_BYTES) + selected = "\n".join(all_lines[s - 1 : e]) text = truncate_text_output( selected, start_line=s, @@ -85,10 +98,26 @@ class ReadStep(BaseStep): max_bytes=max_bytes, file_path=str(target), ) - self.context.response.success = True self.context.response.answer = text self.logger.info( f"[{self.name}] read path={target} lines={s}-{e}/{total} bytes={len(text.encode('utf-8'))}", ) + + async def execute(self): + assert self.context is not None + target = self._resolve_target_or_fail() + if target is None: + return None + if not self._validate_line_params_or_fail(): + return None + content = await self._load_content_or_fail(target) + if content is None: + return None + all_lines = content.split("\n") + rng = self._compute_range_or_fail(target, all_lines) + if rng is None: + return None + s, e, total = rng + self._emit_response(target, all_lines, s, e, total) return self.context.response diff --git a/tests4/unittest/test_crud_md_steps.py b/tests4/unittest/test_crud_md_steps.py new file mode 100644 index 00000000..d60c72f9 --- /dev/null +++ b/tests4/unittest/test_crud_md_steps.py @@ -0,0 +1,371 @@ +"""End-to-end tests for reme4 crud_md steps: spawn `reme4 start`, drive via HTTP, +verify responses, then shut down. Each test uses an isolated cwd so the working_dir +(.reme by default) does not collide. + +CLI rule: `path=` is relative-only, rooted at the reme working_dir. A bare path with +no suffix auto-appends `.md`; non-`.md` suffix is rejected. Absolute paths are +rejected. +""" + +import asyncio +import os +import tempfile +import warnings +from pathlib import Path + +from reme4.utils import call_action, call_and_check, mock_reme_server + +warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources") + + +class _temp_chdir: + """chdir to path for the duration of the block; restore on exit.""" + + def __init__(self, path): + self.path = path + self._old = None + + def __enter__(self): + self._old = os.getcwd() + os.chdir(self.path) + return self + + def __exit__(self, *exc): + os.chdir(self._old) + + +def _run(coro): + """Run an async coroutine on a fresh isolated event loop.""" + asyncio.run(coro) + + +def _seed_md(working_dir: Path, rel: str, body: str) -> Path: + target = working_dir / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body, encoding="utf-8") + return target + + +# --------------------------------------------------------------------------- +# Individual job tests +# --------------------------------------------------------------------------- + + +def test_read_relative_path(): + """`reme4 read path=Templates/Recipe.md` returns the file body from .reme/.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + body = "# Recipe\n\nMix flour and water.\n" + _seed_md(working, "Templates/Recipe.md", body) + async with mock_reme_server() as (host, port): + await call_and_check( + "read", + host=host, + port=port, + path="Templates/Recipe.md", + validator=lambda r: ( + isinstance(r, dict) + and r.get("success") is True + and "# Recipe" in str(r.get("answer", "")) + and "flour and water" in str(r.get("answer", "")) + ), + ) + print("✓ test_read_relative_path passed") + + _run(run()) + + +def test_read_no_suffix_autoappends_md(): + """A bare path with no suffix auto-appends `.md`.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "Templates/Recipe.md", "auto-md\n") + async with mock_reme_server() as (host, port): + await call_and_check( + "read", + host=host, + port=port, + path="Templates/Recipe", + validator=lambda r: ( + isinstance(r, dict) and r.get("success") is True and "auto-md" in str(r.get("answer", "")) + ), + ) + print("✓ test_read_no_suffix_autoappends_md passed") + + _run(run()) + + +def test_read_line_range(): + """start_line / end_line slice the file 1-based, inclusive.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "Notes.md", "L1\nL2\nL3\nL4\nL5\n") + async with mock_reme_server() as (host, port): + await call_and_check( + "read", + host=host, + port=port, + path="Notes.md", + start_line=2, + end_line=4, + validator=lambda r: ( + isinstance(r, dict) + and r.get("success") is True + and "L2" in str(r["answer"]) + and "L3" in str(r["answer"]) + and "L4" in str(r["answer"]) + and "L1" not in str(r["answer"]) + and "L5" not in str(r["answer"]) + ), + ) + print("✓ test_read_line_range passed") + + _run(run()) + + +def test_read_absolute_path_rejected(): + """Absolute paths are rejected (relative-only after refactor).""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + target = _seed_md(working, "Abs.md", "x\n") + async with mock_reme_server() as (host, port): + result = await call_action( + "read", + host=host, + port=port, + path=str(target.resolve()), + ) + if not ( + isinstance(result, dict) + and result.get("success") is False + and "absolute" in str(result.get("answer", "")).lower() + ): + raise AssertionError(f"expected absolute-path rejection, got {result!r}") + print("✓ test_read_absolute_path_rejected passed") + + _run(run()) + + +def test_read_non_md_rejected(): + """Paths whose suffix is not `.md` are rejected.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + async with mock_reme_server() as (host, port): + result = await call_action( + "read", + host=host, + port=port, + path="data/foo.txt", + ) + if not ( + isinstance(result, dict) + and result.get("success") is False + and "markdown" in str(result.get("answer", "")).lower() + ): + raise AssertionError(f"expected markdown-only rejection, got {result!r}") + print("✓ test_read_non_md_rejected passed") + + _run(run()) + + +def test_read_missing_file(): + """Reading a non-existent file should fail with a clear error.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + async with mock_reme_server() as (host, port): + result = await call_action( + "read", + host=host, + port=port, + path="NotThere.md", + ) + if not ( + isinstance(result, dict) + and result.get("success") is False + and "does not exist" in str(result.get("answer", "")).lower() + ): + raise AssertionError(f"expected missing-file rejection, got {result!r}") + print("✓ test_read_missing_file passed") + + _run(run()) + + +def test_read_start_after_end(): + """start_line > end_line is invalid.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "Range.md", "a\nb\nc\n") + async with mock_reme_server() as (host, port): + result = await call_action( + "read", + host=host, + port=port, + path="Range.md", + start_line=3, + end_line=1, + ) + if not ( + isinstance(result, dict) + and result.get("success") is False + and "start_line" in str(result.get("answer", "")) + ): + raise AssertionError(f"expected start>end rejection, got {result!r}") + print("✓ test_read_start_after_end passed") + + _run(run()) + + +def test_read_start_line_exceeds_total(): + """start_line beyond total line count is invalid.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "Short.md", "only-one-line\n") + async with mock_reme_server() as (host, port): + result = await call_action( + "read", + host=host, + port=port, + path="Short.md", + start_line=99, + ) + if not ( + isinstance(result, dict) + and result.get("success") is False + and "exceeds" in str(result.get("answer", "")).lower() + ): + raise AssertionError(f"expected exceeds-length rejection, got {result!r}") + print("✓ test_read_start_line_exceeds_total passed") + + _run(run()) + + +def test_read_truncation(): + """A small max_bytes triggers truncation with a continuation notice.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + body = "\n".join(f"line {i}" for i in range(200)) + "\n" + _seed_md(working, "Big.md", body) + async with mock_reme_server() as (host, port): + await call_and_check( + "read", + host=host, + port=port, + path="Big.md", + max_bytes=64, + validator=lambda r: ( + isinstance(r, dict) + and r.get("success") is True + and "truncated" in str(r["answer"]) + and "start_line=" in str(r["answer"]) + ), + ) + print("✓ test_read_truncation passed") + + _run(run()) + + +def test_read_empty_path_rejected(): + """An empty `path` should be rejected.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + async with mock_reme_server() as (host, port): + result = await call_action("read", host=host, port=port, path="") + if not ( + isinstance(result, dict) + and result.get("success") is False + and "required" in str(result.get("answer", "")).lower() + ): + raise AssertionError(f"expected `path` required rejection, got {result!r}") + print("✓ test_read_empty_path_rejected passed") + + _run(run()) + + +# --------------------------------------------------------------------------- +# Aggregate test: reuse one server instance for all read cases (faster). +# --------------------------------------------------------------------------- + + +def test_all_read_cases_one_server(): + """Run multiple read scenarios against a single shared server for efficiency.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + working = Path(tmp) / ".reme" + working.mkdir(parents=True, exist_ok=True) + _seed_md(working, "Templates/Recipe.md", "# Recipe\nbody\n") + _seed_md(working, "Notes.md", "L1\nL2\nL3\n") + async with mock_reme_server() as (host, port): + await call_and_check( + "read", + host=host, + port=port, + path="Templates/Recipe.md", + validator=lambda r: r.get("success") is True and "# Recipe" in r["answer"], + ) + await call_and_check( + "read", + host=host, + port=port, + path="Notes", + validator=lambda r: r.get("success") is True and "L1" in r["answer"], + ) + await call_and_check( + "read", + host=host, + port=port, + path="Notes.md", + start_line=2, + end_line=2, + validator=lambda r: r.get("success") is True and r["answer"].strip() == "L2", + ) + print("✓ test_all_read_cases_one_server passed") + + _run(run()) + + +if __name__ == "__main__": + print("\n=== reme4 crud_md (read) E2E tests ===") + test_read_relative_path() + test_read_no_suffix_autoappends_md() + test_read_line_range() + test_read_absolute_path_rejected() + test_read_non_md_rejected() + test_read_missing_file() + test_read_start_after_end() + test_read_start_line_exceeds_total() + test_read_truncation() + test_read_empty_path_rejected() + test_all_read_cases_one_server() + print("\n所有测试通过!")