feat(backend): improve workspace support for web clients (#420)

* feat(backend): improve workspace support for web clients

* fix(config): preserve the default workspace directory

* chore(reme): bump version to 0.4.1.5

- Update __version__ from 0.4.1.4 to 0.4.1.5 in initialization file

* fix(chat): disable builtin tools in read-only mode

* fix(agent): make builtin tools opt-in

* fix(list): tolerate files removed during mtime sort

* fix(chat): expose complete read-only job set

reme/__init__.py

@@ -1,6 +1,6 @@
 """ReMe CLI package."""
 
-__version__ = "0.4.1.4"
+__version__ = "0.4.1.5"
 
 from . import config
 from . import constants

reme/components/agent_wrapper/as_agent_wrapper.py

@@ -314,9 +314,9 @@ class AsAgentWrapper(BaseAgentWrapper):
         skills = self._resolve_skills(kwargs.get("skills"))
         tool_context_id = kwargs.get("tool_context_id")
         sequential_tool_calls = bool(kwargs.get("sequential_tool_calls", True))
-        builtin_tools = kwargs.get("builtin_tools", "all")
-        if "builtin_tools" not in kwargs and not bool(kwargs.get("use_builtin_tools", True)):
-            builtin_tools = []
+        builtin_tools = kwargs.get("builtin_tools", [])
+        if "builtin_tools" not in kwargs and bool(kwargs.get("use_builtin_tools", False)):
+            builtin_tools = "all"
         tools: list[ToolBase] = []
         tools.extend(self._builtin_tools(builtin_tools, sequential_tool_calls=sequential_tool_calls))
         tools.extend(self._make_tool(job, tool_context_id, kwargs.get("injected_job_kwargs")) for job in resolved_jobs)

reme/config/default.yaml

@@ -503,6 +503,16 @@ jobs:
           type: boolean
           description: "recurse"
           default: false
+        sort_by:
+          type: string
+          enum:
+            - mtime
+          description: "optional ordering; mtime returns most recently modified files first"
+        extensions:
+          type: array
+          items:
+            type: string
+          description: "optional extension allowlist applied before sorting and limiting"
         limit:
           type: integer
           description: "max results"

reme/schema/application_config.py

@@ -1,6 +1,7 @@
 """Application configuration models."""
 
 import os
+from pathlib import Path
 from pathlib import PurePosixPath, PureWindowsPath
 
 from pydantic import BaseModel, ConfigDict, Field, field_validator
@@ -33,7 +34,11 @@ class ApplicationConfig(BaseModel):
         default_factory=dict,
         description="Environment variables loaded once at startup and passed to agent subprocesses",
     )
-    workspace_dir: str = Field(default=".reme", description="Workspace root directory for runtime files")
+    workspace_dir: str = Field(
+        default=".reme",
+        description="Workspace root directory for runtime files",
+        validate_default=True,
+    )
     metadata_dir: str = Field(default="metadata", description="Subdirectory for ReMe persistent state")
     session_dir: str = Field(default="session", description="Subdirectory for persisted agent sessions")
     # dialog_dir was removed; standard transcripts are always derived as ``{session_dir}/dialog``.
@@ -55,6 +60,12 @@ class ApplicationConfig(BaseModel):
         description="Component registry keyed by type then name",
     )
 
+    @field_validator("workspace_dir", mode="before")
+    @classmethod
+    def normalize_workspace_dir(cls, value) -> str:
+        """Expand home-relative paths once so every component sees the same absolute workspace."""
+        return str(Path(value).expanduser().resolve(strict=False))
+
     @field_validator("session_dir")
     @classmethod
     def validate_session_dir(cls, value: str) -> str:

reme/steps/common/chat.py

@@ -1,5 +1,8 @@
 """Workspace-aware streaming chat step for the ReMe web interface."""
 
+import datetime
+import zoneinfo
+
 from ..base_step import BaseStep
 from ...components import R
 from ...enumeration import ChunkEnum
@@ -13,7 +16,25 @@ class ChatStep(BaseStep):
 Use the available ReMe tools when workspace facts are needed. Cite workspace-relative file paths
 when referring to notes. Never invent file contents, and do not claim to have changed files because
 this chat intentionally provides read-only tools. Reply in the user's language."""
-    READ_ONLY_TOOLS = ["search", "list", "read", "stat", "traverse"]
+    READ_ONLY_TOOLS = ["search", "list", "read", "read_image", "frontmatter_read", "stat", "traverse"]
+
+    def _system_prompt(self) -> str:
+        """Append request-time environment facts to the configured prompt."""
+        timezone = self.app_context.app_config.timezone if self.app_context is not None else None
+        try:
+            current = datetime.datetime.now(zoneinfo.ZoneInfo(timezone)) if timezone else datetime.datetime.now()
+        except (zoneinfo.ZoneInfoNotFoundError, ValueError):
+            current = datetime.datetime.now()
+
+        assert self.context is not None
+        base_prompt = str(self.context.get("system_prompt") or self.DEFAULT_SYSTEM_PROMPT).rstrip()
+        return (
+            f"{base_prompt}\n\n"
+            "<environment_context>\n"
+            f"Current date: {current.date().isoformat()}\n"
+            f"Current working directory: {self.agent_wrapper.cwd.resolve(strict=False)}\n"
+            "</environment_context>"
+        )
 
     async def execute(self):
         assert self.context is not None
@@ -26,14 +47,19 @@ this chat intentionally provides read-only tools. Reply in the user's language."
             raise RuntimeError("chat_step requires an agent_wrapper")
 
         wrapper_kwargs = {
-            "system_prompt": self.context.get("system_prompt") or self.DEFAULT_SYSTEM_PROMPT,
+            "system_prompt": self._system_prompt(),
             "job_tools": self.READ_ONLY_TOOLS,
+            "builtin_tools": [],
         }
         if session_id := self.context.get("session_id"):
             wrapper_kwargs["resume"] = str(session_id)
 
         parts: list[str] = []
         async for chunk in self.agent_wrapper.reply_stream(query, **wrapper_kwargs):
+            if chunk.chunk_type == ChunkEnum.REPLY_END:
+                answer = "".join(parts).strip()
+                if answer:
+                    chunk.metadata["answer"] = answer
             await self.context.add_stream_chunk(chunk)
             if chunk.chunk_type == ChunkEnum.CONTENT and isinstance(chunk.chunk, str):
                 parts.append(chunk.chunk)

reme/steps/file_io/list.py

@@ -8,12 +8,15 @@ Parameters:
     path        — dir to list under (relative to the workspace or absolute). Empty = workspace root.
     limit       — cap on the number of returned items (default 100, must be > 0).
     recursive   — descend into subdirectories. Default False = direct children only.
+    sort_by     — optional ordering; ``mtime`` returns most recently modified files first.
+    extensions  — optional extension allowlist applied before sorting and limiting.
 
 No frontmatter is read. Callers needing frontmatter-based filtering
 should iterate the result and call ``frontmatter_read`` per candidate.
 """
 
 from pathlib import Path
+from stat import S_ISREG
 from typing import Iterable
 
 from ._path import resolve_path
@@ -36,7 +39,7 @@ class ListStep(BaseStep):
         if meta:
             self.context.response.metadata.update(meta)
 
-    def _collect_params(self) -> tuple[str, bool, int]:
+    def _collect_params(self) -> tuple[str, bool, int, str, frozenset[str]]:
         """Read ``path`` / ``recursive`` / ``limit`` from context; coerce permissively."""
         assert self.context is not None
         path = str(self.context.get("path") or "")
@@ -47,20 +50,47 @@ class ListStep(BaseStep):
             limit = int(raw_limit) if raw_limit is not None else DEFAULT_LIMIT
         except (TypeError, ValueError):
             limit = DEFAULT_LIMIT
-        return path, recursive, limit if limit > 0 else DEFAULT_LIMIT
+        sort_by = str(self.context.get("sort_by") or "")
+        raw_extensions = self.context.get("extensions") or []
+        if isinstance(raw_extensions, str):
+            raw_extensions = raw_extensions.split(",")
+        extensions = frozenset(
+            normalized for value in raw_extensions if (normalized := str(value).strip().lower().lstrip("."))
+        )
+        return path, recursive, limit if limit > 0 else DEFAULT_LIMIT, sort_by, extensions
 
     @staticmethod
-    def _walk_files(target_dir: Path, recursive: bool, limit: int) -> list[Path]:
-        """Return up to ``limit`` regular files under ``target_dir``; short-circuits at the cap."""
+    def _walk_files(
+        target_dir: Path,
+        recursive: bool,
+        limit: int,
+        sort_by: str = "",
+        extensions: frozenset[str] = frozenset(),
+    ) -> list[Path]:
+        """Return up to ``limit`` regular files, scanning all entries only when sorting."""
         entries: Iterable[Path] = target_dir.rglob("*") if recursive else target_dir.iterdir()
         files: list[Path] = []
+        mtimes: dict[Path, int] = {}
         for entry in entries:
-            if not entry.is_file():  # skip dirs, sockets, broken links, etc.
+            if sort_by == "mtime":
+                try:
+                    entry_stat = entry.stat()
+                except OSError:
+                    continue
+                if not S_ISREG(entry_stat.st_mode):
+                    continue
+            elif not entry.is_file():  # skip dirs, sockets, broken links, etc.
+                continue
+            if extensions and entry.suffix.lower().lstrip(".") not in extensions:
                 continue
             files.append(entry)
-            if len(files) >= limit:
+            if sort_by == "mtime":
+                mtimes[entry] = entry_stat.st_mtime_ns
+            elif len(files) >= limit:
                 break
-        return files
+        if sort_by == "mtime":
+            files.sort(key=lambda entry: (-mtimes[entry], entry.as_posix()))
+        return files[:limit]
 
     @staticmethod
     def _format_relative(files: list[Path], workspace_dir: Path) -> list[str]:
@@ -75,7 +105,7 @@ class ListStep(BaseStep):
 
     async def execute(self):
         assert self.context is not None
-        path, recursive, limit = self._collect_params()
+        path, recursive, limit, sort_by, extensions = self._collect_params()
         workspace_dir = Path(self.file_store.workspace_path or ".").resolve()
         target_dir, err = resolve_path(workspace_dir, path, allow_empty=True)
         if err or target_dir is None:
@@ -89,7 +119,10 @@ class ListStep(BaseStep):
             self._fail(f"path {target_dir} is not a directory", path=str(target_dir))
             return None
 
-        items = self._format_relative(self._walk_files(target_dir, recursive, limit), workspace_dir)
+        items = self._format_relative(
+            self._walk_files(target_dir, recursive, limit, sort_by, extensions),
+            workspace_dir,
+        )
 
         self.context.response.success = True
         location = path or "."

tests/unit/test_application_config.py

@@ -17,6 +17,15 @@ from reme.steps.index._source_format import is_session_path, render_chunk_body
 from reme.steps.index._watch_rules import build_watch_rules
 
 
+def test_workspace_dir_expands_user_home(monkeypatch, tmp_path):
+    """Home-relative workspace config is normalized before components consume it."""
+    monkeypatch.setenv("HOME", str(tmp_path))
+
+    config = ApplicationConfig(workspace_dir="~/.copaw/workspaces/default")
+
+    assert config.workspace_dir == str(tmp_path / ".copaw/workspaces/default")
+
+
 def test_dialog_dir_is_not_an_application_config_field():
     """The removed option is absent from schemas and ignored when supplied."""
     custom = ApplicationConfig(session_dir="sessions/", dialog_dir="somewhere/else")

tests/unit/test_base_agent_wrapper.py

@@ -1,6 +1,7 @@
 """Tests for shared agent wrapper behavior."""
 
 import sys
+from types import SimpleNamespace
 from unittest.mock import MagicMock
 
 import pytest
@@ -13,6 +14,7 @@ from reme.components.agent_wrapper import (
     handle_session_command,
 )
 from reme.components.agent_wrapper.as_agent_wrapper import WorkspaceBackend
+from reme.components.agent_wrapper import as_agent_wrapper
 from reme.components.agent_wrapper import base_agent_wrapper
 from reme.components.application_context import ApplicationContext
 from reme.components.outbound_proxy import FixedHttpOutboundProxy
@@ -163,3 +165,37 @@ async def test_agentscope_bash_uses_managed_proxy_without_changing_subprocess_en
 
     await wrapper.close()
     await proxy.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("reply_kwargs", "expected"),
+    [
+        ({}, []),
+        ({"use_builtin_tools": True}, "all"),
+        ({"builtin_tools": ["read"]}, ["read"]),
+    ],
+)
+async def test_agentscope_builtin_tools_are_opt_in(tmp_path, monkeypatch, reply_kwargs, expected):
+    """AgentScope loads no built-in tools unless a caller explicitly opts in."""
+    wrapper = AsAgentWrapper(app_context=ApplicationContext(workspace_dir=str(tmp_path)), as_llm="")
+    wrapper.as_llm = SimpleNamespace(model=object())
+    observed = {}
+
+    def builtin_tools(names, *, sequential_tool_calls=False):
+        observed["names"] = names
+        observed["sequential_tool_calls"] = sequential_tool_calls
+        return []
+
+    class FakeAgent:
+        """Minimal constructor double for AgentScope Agent."""
+
+        def __init__(self, **kwargs):
+            self.state = kwargs["state"]
+
+    monkeypatch.setattr(wrapper, "_builtin_tools", builtin_tools)
+    monkeypatch.setattr(as_agent_wrapper, "Agent", FakeAgent)
+
+    await wrapper._build_agent("hello", **reply_kwargs)  # pylint: disable=protected-access
+
+    assert observed == {"names": expected, "sequential_tool_calls": True}

tests/unit/test_crud_steps.py

@@ -227,6 +227,72 @@ def test_list_respects_limit_and_non_recursive():
     asyncio.run(run())
 
 
+def test_list_can_sort_by_most_recent_modification():
+    """Filtering precedes the limit so unrelated generated files cannot hide recent notes."""
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        older = _seed_md(root, "older.md", "old")
+        newer = _seed_md(root, "newer.md", "new")
+        generated = root / "newest.json"
+        generated.write_text("{}", encoding="utf-8")
+        os.utime(older, (100, 100))
+        os.utime(newer, (200, 200))
+        os.utime(generated, (300, 300))
+
+        files = crud_list.ListStep._walk_files(
+            root,
+            recursive=True,
+            limit=2,
+            sort_by="mtime",
+            extensions=frozenset({"md"}),
+        )
+
+        assert [path.name for path in files] == ["newer.md", "older.md"]
+
+
+def test_list_mtime_sort_skips_files_deleted_during_scan(tmp_path):
+    """A disappearing file does not fail the entire sorted listing."""
+    existing = _seed_md(tmp_path, "existing.md", "content")
+
+    class DisappearingEntry:
+        """File-like entry removed before its metadata can be read."""
+
+        suffix = ".md"
+
+        @staticmethod
+        def is_file():
+            """Match the pre-fix scan that first observed a regular file."""
+            return True
+
+        @staticmethod
+        def stat():
+            """Simulate deletion between directory enumeration and metadata lookup."""
+            raise FileNotFoundError("deleted during scan")
+
+        @staticmethod
+        def as_posix():
+            """Return a deterministic path for sorting diagnostics."""
+            return "disappearing.md"
+
+    class Directory:
+        """Directory-like source containing one stable and one vanished entry."""
+
+        @staticmethod
+        def iterdir():
+            """Yield the test entries in filesystem enumeration order."""
+            return iter((existing, DisappearingEntry()))
+
+    files = crud_list.ListStep._walk_files(
+        Directory(),
+        recursive=False,
+        limit=10,
+        sort_by="mtime",
+        extensions=frozenset({"md"}),
+    )
+
+    assert files == [existing]
+
+
 # -- download ------------------------------------------------------------
 #
 # DownloadStep lives in reme_cc (the local plugin overlay), not reme

tests/unit/test_workspace_web_steps.py

@@ -3,6 +3,7 @@
 import asyncio
 import os
 from datetime import datetime
+from zoneinfo import ZoneInfo
 
 from reme.components.agent_wrapper import BaseAgentWrapper
 from reme.components.application_context import ApplicationContext
@@ -17,8 +18,8 @@ from reme.steps.file_io.save import SaveStep
 class _StreamingAgent(BaseAgentWrapper):
     """Minimal agent that records resume/tool arguments and emits rich chunks."""
 
-    def __init__(self):
-        super().__init__()
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
         self.reply_kwargs = {}
 
     async def reply(self, inputs, **kwargs) -> dict:
@@ -28,6 +29,7 @@ class _StreamingAgent(BaseAgentWrapper):
         self.reply_kwargs = kwargs
         yield StreamChunk(chunk_type=ChunkEnum.TOOL_CALL, chunk="{}", tool_call_id="tool-1", tool_call_name="search")
         yield StreamChunk(chunk_type=ChunkEnum.CONTENT, chunk="hello", session_id="session-new")
+        yield StreamChunk(chunk_type=ChunkEnum.REPLY_END, chunk="", session_id="session-new")
 
 
 def test_save_step_preserves_complete_markdown(tmp_path):
@@ -113,21 +115,56 @@ def test_save_step_rejects_external_change(tmp_path):
     asyncio.run(run())
 
 
-def test_chat_step_streams_rich_chunks_and_resumes_session():
+def test_chat_step_streams_rich_chunks_and_resumes_session(tmp_path):
     """Web chat keeps StreamChunk metadata and exposes only read-only tools."""
 
     async def run():
-        agent = _StreamingAgent()
+        app_context = ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai")
+        agent = _StreamingAgent(app_context=app_context)
         queue = asyncio.Queue()
         context = RuntimeContext(stream_queue=queue, query="hello", session_id="session-old")
 
-        response = await ChatStep(agent_wrapper=agent)(context)
+        response = await ChatStep(agent_wrapper=agent, app_context=app_context)(context)
 
-        chunks = [await queue.get(), await queue.get()]
+        chunks = [await queue.get(), await queue.get(), await queue.get()]
         assert response.answer == "hello"
         assert chunks[0].tool_call_name == "search"
         assert chunks[1].session_id == "session-new"
+        assert chunks[2].metadata["answer"] == "hello"
         assert agent.reply_kwargs["resume"] == "session-old"
-        assert agent.reply_kwargs["job_tools"] == ["search", "list", "read", "stat", "traverse"]
+        assert agent.reply_kwargs["job_tools"] == [
+            "search",
+            "list",
+            "read",
+            "read_image",
+            "frontmatter_read",
+            "stat",
+            "traverse",
+        ]
+        assert agent.reply_kwargs["builtin_tools"] == []
+        system_prompt = agent.reply_kwargs["system_prompt"]
+        assert f"Current date: {datetime.now(ZoneInfo('Asia/Shanghai')).date().isoformat()}" in system_prompt
+        assert f"Current working directory: {tmp_path.resolve()}" in system_prompt
+
+    asyncio.run(run())
+
+
+def test_chat_step_appends_environment_context_to_system_prompt_override(tmp_path):
+    """A caller prompt override retains request-time date and cwd context."""
+
+    async def run():
+        app_context = ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai")
+        agent = _StreamingAgent(app_context=app_context)
+
+        await ChatStep(agent_wrapper=agent, app_context=app_context)(
+            query="hello",
+            system_prompt="Custom prompt.",
+            stream_queue=asyncio.Queue(),
+        )
+
+        system_prompt = agent.reply_kwargs["system_prompt"]
+        assert system_prompt.startswith("Custom prompt.\n\n<environment_context>")
+        assert "Current date:" in system_prompt
+        assert f"Current working directory: {tmp_path.resolve()}" in system_prompt
 
     asyncio.run(run())