From c96a192d236cf271d1ab04bb76c848349d65023c Mon Sep 17 00:00:00 2001 From: imrewce Date: Mon, 18 May 2026 19:05:57 +0800 Subject: [PATCH] fix(step): more fixes for pr comments --- docs4/reme_design.md | 4 +-- reme4/config/default.yaml | 10 +++--- reme4/steps/__init__.py | 4 +-- reme4/steps/base_step.py | 23 +++++++------ reme4/steps/{crud_md => crud}/__init__.py | 0 reme4/steps/{crud_md => crud}/_file_io.py | 39 ----------------------- reme4/steps/{crud_md => crud}/read.py | 0 7 files changed, 19 insertions(+), 61 deletions(-) rename reme4/steps/{crud_md => crud}/__init__.py (100%) rename reme4/steps/{crud_md => crud}/_file_io.py (68%) rename reme4/steps/{crud_md => crud}/read.py (100%) diff --git a/docs4/reme_design.md b/docs4/reme_design.md index 06c024ad..fab618ff 100644 --- a/docs4/reme_design.md +++ b/docs4/reme_design.md @@ -42,7 +42,7 @@ reme4 version | 🔎 search | 🔍 `search` (`search_step`) | `call_server("search", query=…, …)` | 📥 `query:str` ⭐ ｜ 🎚️ `limit:int=5`(>0) ｜ 🎚️ `min_score:float=0.0` ｜ ⚖️ `vector_weight:float=0.7` ∈[0,1]（keyword 权 = 1-vw）｜ 🔀 `candidate_multiplier:float=3.0`（candidates = min(200, limit×mult)）｜ 🔗 `expand_links:bool=True` ｜ 🔢 `max_links_per_direction:int=10` ｜ 🎚️ `search_filter:dict={}` ｜ 📤 `answer` 每命中一行 `path:start-end [score=… vector=… keyword=…] text` + 缩进的 `→ outlinks (n)` / `← inlinks (n)` + `via predicate=… anchor=#…` ｜ 📊 `metadata.results` / `metadata.link_expansion` / `metadata.counts={vector,keyword,returned,hybrid}` ｜ 🛠️ 并行 `vector_search` + `keyword_search` → RRF 融合（K=60，按 chunk.id 合并）→ `min_score` 过滤 → `limit` 截断 → 邻居 meta 注入 | | 🧪 demo | 🪄 `demo_echo` (`demo_echo_step1` + `step2`) | `call_server("demo_echo", query=…, min_score=…)` | 📥 `query:str=""` ｜ 🎚️ `min_score:float=0.5` ｜ 🛠️ step1：`processed_query = query.strip().lower()`，`adjusted_min_score = min_score * 0.9`，写回 context ｜ 📤 step2：`answer = "echo: {processed_query} (min_score={adjusted_min_score})"` ｜ 📊 `metadata = {step, query, min_score, processed_query, adjusted_min_score}` | | 🌊 demo | 🌊 `stream_demo` (`stream_demo_step1` + `step2`) | `call_server("stream_demo", query=…, repeat=…, interval=…)` | 📥 `query:str=""` ｜ 🎚️ `repeat:int=10` ｜ 🎚️ `interval:float=0.1`（秒/字符）｜ 🛠️ step1：`stream_text = query * repeat` 写回 context ｜ 📤 step2：按字符 `add_stream_string(ch, ChunkEnum.CONTENT)` 流式输出，`asyncio.sleep(interval)` 节流 | -| 📂 crud_md | 📖 `read` (`read_step`) | `call_server("read", path=…, …)` | 📥 `path:str` ⭐（**相对** `working_dir`，绝对路径会被拒绝；无后缀自动补 `.md`，非 `.md` 后缀拒绝）｜ 🎚️ `start_line:int=null`（1-based, 含端点）｜ 🎚️ `end_line:int=null`（1-based, 含端点）｜ 🎚️ `max_bytes:int=51200`（截断阈值）｜ 📤 `answer = 选中的行内容`，超过 `max_bytes` 时附加 `--- TRUNCATED ---` 续读指引（`start_line=…`）｜ 📊 `metadata.path` / `metadata.total_lines`（出错路径才会附带）｜ 🛠️ 流程：`BaseStep.resolve_path(raw)` → `aiofiles.os.stat` → `read_file_safe`（utf-8-sig BOM 容忍、UnicodeDecodeError fallback `errors=ignore`）→ `split("\n")` 切片 `[s-1:e]` → `truncate_text_output` 按字节截断保行 | +| 📂 crud | 📖 `read` (`read_step`) | `call_server("read", path=…, …)` | 📥 `path:str` ⭐（**完整相对路径**，相对于 `working_dir`；绝对路径会被拒绝；非 `.md` 后缀拒绝）｜ 🎚️ `start_line:int=null`（1-based, 含端点）｜ 🎚️ `end_line:int=null`（1-based, 含端点）｜ 🎚️ `max_bytes:int=51200`（截断阈值）｜ 📤 `answer = 选中的行内容`，超过 `max_bytes` 时附加 `--- TRUNCATED ---` 续读指引（`start_line=…`）｜ 📊 `metadata.path` / `metadata.total_lines`（出错路径才会附带）｜ 🛠️ 流程：`BaseStep.resolve_path(raw, require_md=True)` → `aiofiles.os.stat` → `read_file_safe`（utf-8-sig BOM 容忍、UnicodeDecodeError fallback `errors=ignore`）→ `split("\n")` 切片 `[s-1:e]` → `truncate_text_output` 按字节截断保行 | 使用示例： @@ -67,7 +67,7 @@ reme4 version reme4 reindex reme4 search query="latency 问题" limit=10 min_score=0.2 vector_weight=0.6 -# 读取 working_dir 下的 markdown（相对路径；无后缀自动补 .md；可按行切片或限制字节） +# 读取 working_dir 下的 markdown（完整相对路径；无后缀自动补 .md；可按行切片或限制字节） reme4 read path=Templates/Recipe.md reme4 read path=Notes start_line=1 end_line=20 reme4 read path=Big.md max_bytes=4096 diff --git a/reme4/config/default.yaml b/reme4/config/default.yaml index e4d43568..6e543ec9 100644 --- a/reme4/config/default.yaml +++ b/reme4/config/default.yaml @@ -98,21 +98,19 @@ jobs: - backend: base name: read - description: "read a markdown file (relative to working_dir, no .md suffix needed)" + description: "read a markdown file (relative path under working_dir)" parameters: type: object properties: path: type: string - description: "relative path rooted at the working_dir; markdown only (.md auto-appended when no suffix)" + description: "relative path under the working_dir (no absolute paths); markdown only" start_line: type: integer - description: "first line to read (1-based, inclusive)" - default: null + description: "Optional, first line to read (1-based, inclusive)" end_line: type: integer - description: "last line to read (1-based, inclusive)" - default: null + description: "Optional, last line to read (1-based, inclusive)" required: - path steps: diff --git a/reme4/steps/__init__.py b/reme4/steps/__init__.py index 5cd169fe..70878e5f 100644 --- a/reme4/steps/__init__.py +++ b/reme4/steps/__init__.py @@ -1,11 +1,11 @@ """steps""" from . import common -from . import crud_md +from . import crud from .base_step import BaseStep __all__ = [ "common", - "crud_md", + "crud", "BaseStep", ] diff --git a/reme4/steps/base_step.py b/reme4/steps/base_step.py index b55c079a..2a79a69b 100644 --- a/reme4/steps/base_step.py +++ b/reme4/steps/base_step.py @@ -91,15 +91,15 @@ class BaseStep(ABC): self, raw: str, *, - require_md: bool = True, + require_md: bool = False, ) -> tuple[Path | None, str | None]: - """Resolve a relative `path=` argument under self.working_path. + """Resolve relative `path=` argument under self.working_path. Rules: - - relative path only; joined under ``self.working_path``. - Markdown gate (when ``require_md=True``): - - no suffix → auto-append ``.md``; - - any other non-``.md`` suffix → reject. + - the caller supplies the full relative path under ``self.working_path``; + absolute paths are rejected. + - if the path has no suffix, auto-append ``.md`` (default vault extension). + - if ``require_md=True``, any present non-``.md`` suffix is rejected. Returns ``(abs_path, None)`` on success, or ``(None, error_message)`` on failure. """ if not raw or not str(raw).strip(): @@ -107,13 +107,12 @@ class BaseStep(ABC): s = str(raw).strip() p = Path(s) if p.is_absolute(): - return None, (f"path {s!r} is absolute; only relative paths under the working_dir are accepted") + return None, (f"path {s!r} is absolute; only relative paths accepted") target = (self.working_path.resolve() / p).resolve() - if require_md: - if target.suffix == "": - target = target.with_suffix(".md") - elif target.suffix.lower() != ".md": - return None, (f"path {s!r} is not a markdown file; this command only supports .md files") + if target.suffix == "": + target = target.with_suffix(".md") + elif require_md and target.suffix.lower() != ".md": + return None, (f"path {s!r} is not a markdown file; this command only supports .md files") return target, None def _resolve( diff --git a/reme4/steps/crud_md/__init__.py b/reme4/steps/crud/__init__.py similarity index 100% rename from reme4/steps/crud_md/__init__.py rename to reme4/steps/crud/__init__.py diff --git a/reme4/steps/crud_md/_file_io.py b/reme4/steps/crud/_file_io.py similarity index 68% rename from reme4/steps/crud_md/_file_io.py rename to reme4/steps/crud/_file_io.py index 44759ff9..e2d14ce0 100644 --- a/reme4/steps/crud_md/_file_io.py +++ b/reme4/steps/crud/_file_io.py @@ -1,7 +1,5 @@ """Shared filesystem helpers for CRUD steps (safe read, truncation).""" -import re - import aiofiles import aiofiles.os @@ -47,9 +45,6 @@ def truncate_text_output( return text try: - if TRUNCATION_NOTICE_MARKER in text: - return _retruncate(text, max_bytes=max_bytes, encoding=encoding) - text_bytes = text.encode(encoding) if len(text_bytes) <= max_bytes: return text @@ -77,37 +72,3 @@ def truncate_text_output( except Exception: logger.warning("truncate_text_output failed, returning original text", exc_info=True) return text - - -def _retruncate(text: str, *, max_bytes: int, encoding: str) -> str: - parts = text.split(TRUNCATION_NOTICE_MARKER, 1) - original_content = parts[0] - old_notice = parts[1] - - text_bytes = original_content.encode(encoding) - if len(text_bytes) <= max_bytes + 100: - return text - - start_match = re.search(r"starts at line (\d+)", old_notice) - if not start_match: - return text - start_line_parsed = int(start_match.group(1)) - - truncated_bytes = text_bytes[:max_bytes] - result = truncated_bytes.decode(encoding, errors="ignore") - newline_count = result.count("\n") - next_line = start_line_parsed + max(1, newline_count) - - if not re.search(r"covers the next \d+ bytes", old_notice): - return text - new_notice = re.sub( - r"covers the next \d+ bytes", - f"covers the next {max_bytes} bytes", - old_notice, - ) - new_notice = re.sub( - r"start_line=\d+ to read more", - f"start_line={next_line} to read more", - new_notice, - ) - return result + TRUNCATION_NOTICE_MARKER + new_notice diff --git a/reme4/steps/crud_md/read.py b/reme4/steps/crud/read.py similarity index 100% rename from reme4/steps/crud_md/read.py rename to reme4/steps/crud/read.py