feat: implementation of the read step for reme (markdown)

reme4/config/default.yaml

@@ -96,6 +96,30 @@ jobs:
     steps:
       - backend: search_step
 
+  - backend: base
+    name: read
+    description: "read a markdown file (absolute path; relative path resolved under .reme)"
+    parameters:
+      type: object
+      properties:
+        path:
+          type: string
+          description: "absolute or relative path; relative is rooted at the reme working_dir (.reme); markdown only (.md auto-appended when no suffix)"
+        start_line:
+          type: integer
+          description: "first line to read (1-based, inclusive)"
+        end_line:
+          type: integer
+          description: "last line to read (1-based, inclusive)"
+        max_bytes:
+          type: integer
+          description: "max bytes returned before byte-level truncation"
+          default: 51200
+      required:
+        - path
+    steps:
+      - backend: read_step
+
   - backend: stream
     name: stream_demo
     description: "stream demo job: repeat query 10x and stream char-by-char"

reme4/constants.py

@@ -5,3 +5,8 @@ REME_SERVICE_INFO = "REME_SERVICE_INFO"
 REME_DEFAULT_HOST = "127.0.0.1"
 
 REME_DEFAULT_PORT = 2333
+
+# CRUD steps: file IO limits and truncation marker (shared across CRUD steps).
+DEFAULT_MAX_BYTES = 50 * 1024
+MAX_FILE_READ_BYTES = 200 * 1024 * 1024
+TRUNCATION_NOTICE_MARKER = "<<TRUNCATION_NOTICE>>"

reme4/steps/common/__init__.py

@@ -3,6 +3,7 @@
 from .demo import DemoEchoStep1, DemoEchoStep2
 from .health_check import HealthCheckStep
 from .help import HelpStep
+from .read import ReadStep
 from .reindex import ReindexStep
 from .search import SearchStep
 from .stream_demo import StreamDemoStep1, StreamDemoStep2
@@ -13,6 +14,7 @@ __all__ = [
     "DemoEchoStep2",
     "HealthCheckStep",
     "HelpStep",
+    "ReadStep",
     "ReindexStep",
     "SearchStep",
     "StreamDemoStep1",

reme4/steps/common/_file_io.py

@@ -0,0 +1,159 @@
+"""Shared filesystem helpers for CRUD steps (path resolve, safe read, truncation)."""
+
+import logging
+import re
+from pathlib import Path
+
+import aiofiles
+import aiofiles.os
+
+from ...constants import DEFAULT_MAX_BYTES, MAX_FILE_READ_BYTES, TRUNCATION_NOTICE_MARKER
+
+logger = logging.getLogger(__name__)
+
+
+def resolve_path(
+    working_path: Path,
+    raw: str,
+    *,
+    require_md: bool = True,
+) -> tuple[Path | None, str | None]:
+    """Resolve a `path=` argument into an absolute Path (qwenpaw-style).
+
+    Rules:
+      - absolute path  → used as-is (with `~` expansion);
+      - relative path  → joined under ``working_path`` (the reme working_dir, default ``.reme``).
+    Markdown gate (when ``require_md=True``):
+      - no suffix (e.g. ``My Note``) → auto-append ``.md``;
+      - any other non-``.md`` suffix → reject.
+    Returns ``(abs_path, None)`` on success, or ``(None, error_message)`` on failure.
+    """
+    if not raw or not str(raw).strip():
+        return None, "`path` is required"
+    s = str(raw).strip()
+    p = Path(s).expanduser()
+    target = p.resolve() if p.is_absolute() else (working_path / p).resolve()
+    if require_md:
+        if target.suffix == "":
+            target = target.with_suffix(".md")
+        elif target.suffix.lower() != ".md":
+            return None, (
+                f"path {s!r} is not a markdown file; this command only supports .md files"
+            )
+    return target, None
+
+
+async def read_file_safe(file_path: Path | str, max_bytes: int = MAX_FILE_READ_BYTES) -> str:
+    """Read file with utf-8-sig (BOM-tolerant), fallback to errors='ignore'."""
+    stat = await aiofiles.os.stat(str(file_path))
+    read_size = min(stat.st_size, max_bytes)
+    try:
+        async with aiofiles.open(str(file_path), "r", encoding="utf-8-sig") as f:
+            return await f.read(read_size)
+    except UnicodeDecodeError:
+        async with aiofiles.open(
+            str(file_path), "r", encoding="utf-8-sig", errors="ignore",
+        ) as f:
+            return await f.read(read_size)
+
+
+def truncate_text_output(
+    text: str,
+    *,
+    start_line: int = 1,
+    total_lines: int = 0,
+    max_bytes: int = DEFAULT_MAX_BYTES,
+    file_path: str | None = None,
+    encoding: str = "utf-8",
+) -> str:
+    """Truncate text by bytes preserving line integrity; append a continuation notice.
+
+    See qwenpaw `tools/utils.py` for the same semantics. Returns text unchanged when
+    it fits within max_bytes, when max_bytes <= 0, or when the last line itself
+    exceeds max_bytes (unhandled edge case).
+    """
+    if not text or max_bytes <= 0:
+        return text
+
+    try:
+        if TRUNCATION_NOTICE_MARKER in text:
+            return _retruncate(text, max_bytes=max_bytes, encoding=encoding)
+        return _truncate_fresh(
+            text,
+            start_line=start_line,
+            total_lines=total_lines,
+            max_bytes=max_bytes,
+            file_path=file_path,
+            encoding=encoding,
+        )
+    except Exception:
+        logger.warning("truncate_text_output failed, returning original text", exc_info=True)
+        return text
+
+
+def _truncate_fresh(
+    text: str,
+    *,
+    start_line: int,
+    total_lines: int,
+    max_bytes: int,
+    file_path: str | None,
+    encoding: str,
+) -> str:
+    text_bytes = text.encode(encoding)
+    if len(text_bytes) <= max_bytes:
+        return text
+
+    truncated = text_bytes[:max_bytes]
+    result = truncated.decode(encoding, errors="ignore")
+    newline_count = result.count("\n")
+    next_line = start_line + max(1, newline_count)
+
+    if next_line <= total_lines:
+        read_from = next_line
+    elif start_line < total_lines:
+        read_from = total_lines
+    else:
+        return result
+
+    notice = (
+        TRUNCATION_NOTICE_MARKER
+        + f"\nThe output above was truncated."
+        f"\nThe full content is saved to the file and contains {total_lines} lines in total."
+        f"\nThis excerpt starts at line {start_line} and covers the next {max_bytes} bytes."
+        f"\nIf the current content is not enough, call `read` with file={file_path or ''} "
+        f"start_line={read_from} to read more."
+    )
+    return result + notice
+
+
+def _retruncate(text: str, *, max_bytes: int, encoding: str) -> str:
+    parts = text.split(TRUNCATION_NOTICE_MARKER, 1)
+    original_content = parts[0]
+    old_notice = parts[1]
+
+    text_bytes = original_content.encode(encoding)
+    if len(text_bytes) <= max_bytes + 100:
+        return text
+
+    start_match = re.search(r"starts at line (\d+)", old_notice)
+    if not start_match:
+        return text
+    start_line_parsed = int(start_match.group(1))
+
+    truncated_bytes = text_bytes[:max_bytes]
+    result = truncated_bytes.decode(encoding, errors="ignore")
+    newline_count = result.count("\n")
+    next_line = start_line_parsed + max(1, newline_count)
+
+    if not re.search(r"covers the next \d+ bytes", old_notice):
+        return text
+    new_notice = re.sub(
+        r"covers the next \d+ bytes", f"covers the next {max_bytes} bytes", old_notice,
+    )
+    new_notice = re.sub(
+        r"start_line=\d+ to read more", f"start_line={next_line} to read more", new_notice,
+    )
+    return result + TRUNCATION_NOTICE_MARKER + new_notice
+
+

reme4/steps/common/read.py

@@ -0,0 +1,94 @@
+"""Read a markdown file from the vault, with line-range slicing and byte-truncation."""
+
+from pathlib import Path
+
+from ._file_io import (
+    DEFAULT_MAX_BYTES,
+    read_file_safe,
+    resolve_path,
+    truncate_text_output,
+)
+from ..base_step import BaseStep
+from ...components import R
+
+
+@R.register("read_step")
+class ReadStep(BaseStep):
+    """Read a markdown file. Optional `start_line`/`end_line` for ranged reads."""
+
+    def _working_path(self) -> Path:
+        if self.app_context is not None:
+            return Path(self.app_context.app_config.working_dir).resolve()
+        return Path.cwd().resolve()
+
+    def _fail(self, message: str, **meta) -> None:
+        assert self.context is not None
+        self.context.response.success = False
+        self.context.response.answer = f"Error: {message}"
+        if meta:
+            self.context.response.metadata.update(meta)
+
+    async def execute(self):
+        assert self.context is not None
+        raw = str(self.context.get("path") or "")
+        start_line = self.context.get("start_line")
+        end_line = self.context.get("end_line")
+        max_bytes = int(self.context.get("max_bytes") or DEFAULT_MAX_BYTES)
+
+        target, err = resolve_path(self._working_path(), raw, require_md=True)
+        if err:
+            self._fail(err)
+            return
+
+        for label, value in (("start_line", start_line), ("end_line", end_line)):
+            if value is None:
+                continue
+            try:
+                int(value)
+            except (TypeError, ValueError):
+                self._fail(f"{label} must be an integer, got {value!r}")
+                return
+
+        if not target.exists():
+            self._fail(f"file {target} does not exist", path=str(target))
+            return
+        if not target.is_file():
+            self._fail(f"path {target} is not a file", path=str(target))
+            return
+
+        try:
+            content = await read_file_safe(target)
+        except Exception as e:
+            self._fail(f"read failed: {e}", path=str(target))
+            return
+
+        all_lines = content.split("\n")
+        total = len(all_lines)
+        s = max(1, int(start_line) if start_line is not None else 1)
+        e = min(total, int(end_line) if end_line is not None else total)
+
+        if s > total:
+            self._fail(
+                f"start_line {s} exceeds file length ({total} lines)",
+                path=str(target), total_lines=total,
+            )
+            return
+        if s > e:
+            self._fail(f"start_line ({s}) > end_line ({e})", path=str(target))
+            return
+
+        selected = "\n".join(all_lines[s - 1: e])
+        text = truncate_text_output(
+            selected,
+            start_line=s,
+            total_lines=total,
+            max_bytes=max_bytes,
+            file_path=str(target),
+        )
+
+        self.context.response.success = True
+        self.context.response.answer = text
+        self.logger.info(
+            f"[{self.name}] read path={target} lines={s}-{e}/{total} bytes={len(text.encode('utf-8'))}",
+        )
+        return self.context.response