From bbfc4a4acc6800b45a3ba01db6bfae941959a78a Mon Sep 17 00:00:00 2001 From: imrewce Date: Mon, 18 May 2026 16:05:23 +0800 Subject: [PATCH] feat: implementation of the read step for reme (markdown) --- reme4/config/default.yaml | 24 +++++ reme4/constants.py | 5 ++ reme4/steps/common/__init__.py | 2 + reme4/steps/common/_file_io.py | 159 +++++++++++++++++++++++++++++++++ reme4/steps/common/read.py | 94 +++++++++++++++++++ 5 files changed, 284 insertions(+) create mode 100644 reme4/steps/common/_file_io.py create mode 100644 reme4/steps/common/read.py diff --git a/reme4/config/default.yaml b/reme4/config/default.yaml index 1a501912..e06cf5c4 100644 --- a/reme4/config/default.yaml +++ b/reme4/config/default.yaml @@ -96,6 +96,30 @@ jobs: steps: - backend: search_step + - backend: base + name: read + description: "read a markdown file (absolute path; relative path resolved under .reme)" + parameters: + type: object + properties: + path: + type: string + description: "absolute or relative path; relative is rooted at the reme working_dir (.reme); markdown only (.md auto-appended when no suffix)" + start_line: + type: integer + description: "first line to read (1-based, inclusive)" + end_line: + type: integer + description: "last line to read (1-based, inclusive)" + max_bytes: + type: integer + description: "max bytes returned before byte-level truncation" + default: 51200 + required: + - path + steps: + - backend: read_step + - backend: stream name: stream_demo description: "stream demo job: repeat query 10x and stream char-by-char" diff --git a/reme4/constants.py b/reme4/constants.py index fce66e64..819fbadd 100644 --- a/reme4/constants.py +++ b/reme4/constants.py @@ -5,3 +5,8 @@ REME_SERVICE_INFO = "REME_SERVICE_INFO" REME_DEFAULT_HOST = "127.0.0.1" REME_DEFAULT_PORT = 2333 + +# CRUD steps: file IO limits and truncation marker (shared across CRUD steps). +DEFAULT_MAX_BYTES = 50 * 1024 +MAX_FILE_READ_BYTES = 200 * 1024 * 1024 +TRUNCATION_NOTICE_MARKER = "<>" diff --git a/reme4/steps/common/__init__.py b/reme4/steps/common/__init__.py index e09c2073..bda17e23 100644 --- a/reme4/steps/common/__init__.py +++ b/reme4/steps/common/__init__.py @@ -3,6 +3,7 @@ from .demo import DemoEchoStep1, DemoEchoStep2 from .health_check import HealthCheckStep from .help import HelpStep +from .read import ReadStep from .reindex import ReindexStep from .search import SearchStep from .stream_demo import StreamDemoStep1, StreamDemoStep2 @@ -13,6 +14,7 @@ __all__ = [ "DemoEchoStep2", "HealthCheckStep", "HelpStep", + "ReadStep", "ReindexStep", "SearchStep", "StreamDemoStep1", diff --git a/reme4/steps/common/_file_io.py b/reme4/steps/common/_file_io.py new file mode 100644 index 00000000..ea8b8d80 --- /dev/null +++ b/reme4/steps/common/_file_io.py @@ -0,0 +1,159 @@ +"""Shared filesystem helpers for CRUD steps (path resolve, safe read, truncation).""" + +import logging +import re +from pathlib import Path + +import aiofiles +import aiofiles.os + +from ...constants import DEFAULT_MAX_BYTES, MAX_FILE_READ_BYTES, TRUNCATION_NOTICE_MARKER + +logger = logging.getLogger(__name__) + + +def resolve_path( + working_path: Path, + raw: str, + *, + require_md: bool = True, +) -> tuple[Path | None, str | None]: + """Resolve a `path=` argument into an absolute Path (qwenpaw-style). + + Rules: + - absolute path → used as-is (with `~` expansion); + - relative path → joined under ``working_path`` (the reme working_dir, default ``.reme``). + Markdown gate (when ``require_md=True``): + - no suffix (e.g. ``My Note``) → auto-append ``.md``; + - any other non-``.md`` suffix → reject. + Returns ``(abs_path, None)`` on success, or ``(None, error_message)`` on failure. + """ + if not raw or not str(raw).strip(): + return None, "`path` is required" + s = str(raw).strip() + p = Path(s).expanduser() + target = p.resolve() if p.is_absolute() else (working_path / p).resolve() + if require_md: + if target.suffix == "": + target = target.with_suffix(".md") + elif target.suffix.lower() != ".md": + return None, ( + f"path {s!r} is not a markdown file; this command only supports .md files" + ) + return target, None + + +async def read_file_safe(file_path: Path | str, max_bytes: int = MAX_FILE_READ_BYTES) -> str: + """Read file with utf-8-sig (BOM-tolerant), fallback to errors='ignore'.""" + stat = await aiofiles.os.stat(str(file_path)) + read_size = min(stat.st_size, max_bytes) + try: + async with aiofiles.open(str(file_path), "r", encoding="utf-8-sig") as f: + return await f.read(read_size) + except UnicodeDecodeError: + async with aiofiles.open( + str(file_path), "r", encoding="utf-8-sig", errors="ignore", + ) as f: + return await f.read(read_size) + + +def truncate_text_output( + text: str, + *, + start_line: int = 1, + total_lines: int = 0, + max_bytes: int = DEFAULT_MAX_BYTES, + file_path: str | None = None, + encoding: str = "utf-8", +) -> str: + """Truncate text by bytes preserving line integrity; append a continuation notice. + + See qwenpaw `tools/utils.py` for the same semantics. Returns text unchanged when + it fits within max_bytes, when max_bytes <= 0, or when the last line itself + exceeds max_bytes (unhandled edge case). + """ + if not text or max_bytes <= 0: + return text + + try: + if TRUNCATION_NOTICE_MARKER in text: + return _retruncate(text, max_bytes=max_bytes, encoding=encoding) + return _truncate_fresh( + text, + start_line=start_line, + total_lines=total_lines, + max_bytes=max_bytes, + file_path=file_path, + encoding=encoding, + ) + except Exception: + logger.warning("truncate_text_output failed, returning original text", exc_info=True) + return text + + +def _truncate_fresh( + text: str, + *, + start_line: int, + total_lines: int, + max_bytes: int, + file_path: str | None, + encoding: str, +) -> str: + text_bytes = text.encode(encoding) + if len(text_bytes) <= max_bytes: + return text + + truncated = text_bytes[:max_bytes] + result = truncated.decode(encoding, errors="ignore") + newline_count = result.count("\n") + next_line = start_line + max(1, newline_count) + + if next_line <= total_lines: + read_from = next_line + elif start_line < total_lines: + read_from = total_lines + else: + return result + + notice = ( + TRUNCATION_NOTICE_MARKER + + f"\nThe output above was truncated." + f"\nThe full content is saved to the file and contains {total_lines} lines in total." + f"\nThis excerpt starts at line {start_line} and covers the next {max_bytes} bytes." + f"\nIf the current content is not enough, call `read` with file={file_path or ''} " + f"start_line={read_from} to read more." + ) + return result + notice + + +def _retruncate(text: str, *, max_bytes: int, encoding: str) -> str: + parts = text.split(TRUNCATION_NOTICE_MARKER, 1) + original_content = parts[0] + old_notice = parts[1] + + text_bytes = original_content.encode(encoding) + if len(text_bytes) <= max_bytes + 100: + return text + + start_match = re.search(r"starts at line (\d+)", old_notice) + if not start_match: + return text + start_line_parsed = int(start_match.group(1)) + + truncated_bytes = text_bytes[:max_bytes] + result = truncated_bytes.decode(encoding, errors="ignore") + newline_count = result.count("\n") + next_line = start_line_parsed + max(1, newline_count) + + if not re.search(r"covers the next \d+ bytes", old_notice): + return text + new_notice = re.sub( + r"covers the next \d+ bytes", f"covers the next {max_bytes} bytes", old_notice, + ) + new_notice = re.sub( + r"start_line=\d+ to read more", f"start_line={next_line} to read more", new_notice, + ) + return result + TRUNCATION_NOTICE_MARKER + new_notice + + diff --git a/reme4/steps/common/read.py b/reme4/steps/common/read.py new file mode 100644 index 00000000..6db8fb4d --- /dev/null +++ b/reme4/steps/common/read.py @@ -0,0 +1,94 @@ +"""Read a markdown file from the vault, with line-range slicing and byte-truncation.""" + +from pathlib import Path + +from ._file_io import ( + DEFAULT_MAX_BYTES, + read_file_safe, + resolve_path, + truncate_text_output, +) +from ..base_step import BaseStep +from ...components import R + + +@R.register("read_step") +class ReadStep(BaseStep): + """Read a markdown file. Optional `start_line`/`end_line` for ranged reads.""" + + def _working_path(self) -> Path: + if self.app_context is not None: + return Path(self.app_context.app_config.working_dir).resolve() + return Path.cwd().resolve() + + def _fail(self, message: str, **meta) -> None: + assert self.context is not None + self.context.response.success = False + self.context.response.answer = f"Error: {message}" + if meta: + self.context.response.metadata.update(meta) + + async def execute(self): + assert self.context is not None + raw = str(self.context.get("path") or "") + start_line = self.context.get("start_line") + end_line = self.context.get("end_line") + max_bytes = int(self.context.get("max_bytes") or DEFAULT_MAX_BYTES) + + target, err = resolve_path(self._working_path(), raw, require_md=True) + if err: + self._fail(err) + return + + for label, value in (("start_line", start_line), ("end_line", end_line)): + if value is None: + continue + try: + int(value) + except (TypeError, ValueError): + self._fail(f"{label} must be an integer, got {value!r}") + return + + if not target.exists(): + self._fail(f"file {target} does not exist", path=str(target)) + return + if not target.is_file(): + self._fail(f"path {target} is not a file", path=str(target)) + return + + try: + content = await read_file_safe(target) + except Exception as e: + self._fail(f"read failed: {e}", path=str(target)) + return + + all_lines = content.split("\n") + total = len(all_lines) + s = max(1, int(start_line) if start_line is not None else 1) + e = min(total, int(end_line) if end_line is not None else total) + + if s > total: + self._fail( + f"start_line {s} exceeds file length ({total} lines)", + path=str(target), total_lines=total, + ) + return + if s > e: + self._fail(f"start_line ({s}) > end_line ({e})", path=str(target)) + return + + selected = "\n".join(all_lines[s - 1: e]) + text = truncate_text_output( + selected, + start_line=s, + total_lines=total, + max_bytes=max_bytes, + file_path=str(target), + ) + + self.context.response.success = True + self.context.response.answer = text + self.logger.info( + f"[{self.name}] read path={target} lines={s}-{e}/{total} bytes={len(text.encode('utf-8'))}", + ) + return self.context.response