From c3fb825af07f31e9a1b35b662b7fb0dd15da218f Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Mon, 8 Jun 2026 16:11:32 +0800 Subject: [PATCH] feat(agent): refactor agent wrapper, add session persistence, auto_resource step, and watch-loop improvements (#277) * refactor(agent_wrapper): update agent wrapper implementations and config defaults - Set default timezone to Asia/Shanghai in application config - Add AgentScope imports and configure ReAct, context, and model configs - Simplify __all__ export formatting in agent wrapper init - Remove redundant docstring details from agent wrapper classes - Optimize tool result handling with state assignment simplification - Add permission context and state management for AgentScope backend - Update Claude Code wrapper tool creation and server registration logic - Configure default agent settings including permission mode and retry limits - Remove obsolete comments and streamline code structure * fix(agent): add output schema validation and BaseModel support - Added type assertion to ensure output_schema is a dict in as_agent_wrapper - Imported BaseModel from pydantic in base_agent_wrapper - Modified set_output_schema to accept both dict and BaseModel types - Added automatic conversion of BaseModel to JSON schema - Updated method documentation to reflect new type support * refactor(agent): replace direct agent instantiation with agent wrapper component - Removed manual Agent creation and initialization in llm_demo step - Integrated agent_wrapper component as dependency in base step - Updated llm_demo step to use agent_wrapper.reply method instead of direct agent calls - Modified structured output handling to work with new agent wrapper interface - Simplified agent configuration by using wrapper's built-in functionality - Updated documentation to reflect agent wrapper usage instead of direct as_llm access - Removed redundant imports related to manual agent management * feat(agent): add streaming support and refactor agent wrapper components - Introduce reply_stream method in base agent wrapper with fallback implementation - Add _build_agent helper method to AsAgentWrapper for agent instantiation - Implement structured output generation with proper model assertions - Update StreamLLMDemoStep to use agent_wrapper instead of direct Agent calls - Replace manual streaming logic with execute_stream_task utility function - Change default system prompt to provide detailed responses instead of concise ones - Add colored output support for different chunk types in streaming demos - Refactor test cases to use async task execution with streaming verification * refactor(agent): remove session_id parameter from reply methods - Removed session_id parameter from ASAgentWrapper.reply method signature - Removed session_id parameter from BaseAgentWrapper.reply abstract method - Removed session_id parameter from CCAgentWrapper.reply method signature - Updated reply_stream methods to remove session_id parameter across all wrappers - Modified CCAgentWrapper to use dynamic options assignment instead of hardcoded properties - Set default system_prompt in config instead of hardcoded in code - Increased default max_turns from 10 to 50 in configuration * config: update default configuration and script entry point - Change resource_dir from empty string to 'resource' - Update command line entry point from 'reme4' to 'reme' * feat(agent): add session state persistence and forking support - Implement AsStateHandler for AgentState JSONL serialization - Add session_id parameter to AsAgentWrapper.reply method - Create timestamp-based session file paths with timezone support - Load existing session state from JSONL files when session_id provided - Save updated session state after each agent interaction - Support session forking with UUID generation for new sessions - Add integration tests for session persistence and forking scenarios - Include temporary directory utilities for testing isolated sessions - Ensure parent directories are created for session files automatically * refactor(auto_memory): replace transcript parsing with direct message handling - Remove transcript loading logic and related dependencies - Add session message saving functionality with deduplication - Use agent wrapper instead of direct AgentScope agent instantiation - Simplify timezone handling using shared now utility - Update logging and response metadata structure - Remove unused imports and toolkit management methods - Change session file naming from session_{id}.jsonl to session_agent_{id}.jsonl * refactor(steps): move channel steps from index to channel module - Move ChannelNotifyStep from .index.channel_notify to .channel.channel_notify - Move ClaimChannelStep from .index.claim_channel to .channel.claim_channel - Update __init__.py imports to reflect new module structure - Reorganize steps list in __init__.py with channel section before index - Add proper file prefix handling in daily index processing - Update test imports to use new channel module location * feat(evolve): add auto_resource step for interpreting resource files - Add AutoResourceStep to interpret resource files into daily notes via an agent - Implement resource file parsing with date and filename extraction logic - Add session ID computation using MD5 hash of filename - Create delete and upsert handlers for resource file operations - Add truncation and sanitization functions for tool output in auto_memory - Register auto_resource step with proper parameter validation - Add configuration for resource watch loop with file extension filters - Update default YAML config to include resource watch and digest watch loops - Add shared watch-rule logic for scan_changes and watch_changes steps - Implement foreach_dispatch and log_changes steps for change processing - Rename update_store_index_loop to index_update_loop in configuration - Refactor file chunking interface from parse to chunk method - Remove unused imports and dependencies in auto_dream step - Fix path iteration formatting in daily_index utility function - Add comprehensive integration tests for auto_resource functionality * refactor(auto_resource): format function call with multi-line parameters - Reformatted await _handle_upsert call to use multiple lines for better readability - Removed unused imports from scan_changes.py including BaseFileCatalog and ComponentEnum - Added date parameter to RuntimeContext initialization in test cases - Updated expected file paths in test assertions to include session_agent prefix - Formatted long assertion statements across multiple lines to maintain character limit - Corrected wikilink references from generic names to session_agent prefixed names --- docs4/old/reme4_resource.md | 2 +- reme4/components/agent_wrapper/__init__.py | 6 +- .../agent_wrapper/as_agent_wrapper.py | 118 ++-- .../agent_wrapper/base_agent_wrapper.py | 28 +- .../agent_wrapper/cc_agent_wrapper.py | 49 +- .../file_chunker/base_file_chunker.py | 6 +- .../file_chunker/default_file_chunker.py | 2 +- .../file_chunker/markdown_file_chunker.py | 2 +- reme4/components/job/base_job.py | 2 - reme4/config/default.yaml | 68 ++- reme4/pyproject.toml | 2 +- reme4/schema/application_config.py | 2 +- reme4/steps/__init__.py | 13 +- reme4/steps/base_step.py | 62 +-- reme4/steps/channel/__init__.py | 0 .../{index => channel}/channel_notify.py | 0 .../steps/{index => channel}/claim_channel.py | 0 reme4/steps/common/llm_demo.py | 47 +- reme4/steps/common/stream_llm_demo.py | 42 +- reme4/steps/evolve/auto_dream.py | 9 +- reme4/steps/evolve/auto_memory.py | 209 ++++--- reme4/steps/evolve/auto_resource.py | 144 +++++ reme4/steps/evolve/auto_resource.yaml | 218 ++++++++ reme4/steps/evolve/dream.py | 121 +--- reme4/steps/file_io/_daily_index.py | 9 +- reme4/steps/file_io/daily_create.py | 2 +- reme4/steps/index/_watch_rules.py | 68 +++ reme4/steps/index/foreach_dispatch.py | 27 + reme4/steps/index/log_changes.py | 18 + reme4/steps/index/scan_changes.py | 77 +-- reme4/steps/index/update_catalog.py | 11 +- reme4/steps/index/update_index.py | 33 +- reme4/steps/index/watch_changes.py | 60 +- reme4/utils/__init__.py | 2 + reme4/utils/agent_state_io.py | 54 ++ reme4/utils/transcript.py | 195 ------- tests4/integration/test_agent_session.py | 170 ++++++ tests4/integration/test_auto_memory.py | 3 +- tests4/integration/test_auto_resource.py | 361 ++++++++++++ tests4/integration/test_stream_llm.py | 148 +++-- tests4/unit/test_auto_dream.py | 18 +- tests4/unit/test_background_steps.py | 520 +++++++++++++----- tests4/unit/test_channel_notify.py | 2 +- tests4/unit/test_daily_steps.py | 58 +- tests4/unit/test_default_file_chunker.py | 58 +- tests4/unit/test_markdown_file_chunker.py | 44 +- 46 files changed, 2129 insertions(+), 961 deletions(-) create mode 100644 reme4/steps/channel/__init__.py rename reme4/steps/{index => channel}/channel_notify.py (100%) rename reme4/steps/{index => channel}/claim_channel.py (100%) create mode 100644 reme4/steps/evolve/auto_resource.py create mode 100644 reme4/steps/evolve/auto_resource.yaml create mode 100644 reme4/steps/index/_watch_rules.py create mode 100644 reme4/steps/index/foreach_dispatch.py create mode 100644 reme4/steps/index/log_changes.py create mode 100644 reme4/utils/agent_state_io.py delete mode 100644 reme4/utils/transcript.py create mode 100644 tests4/integration/test_agent_session.py create mode 100644 tests4/integration/test_auto_resource.py diff --git a/docs4/old/reme4_resource.md b/docs4/old/reme4_resource.md index 6ce5e481..44020210 100644 --- a/docs4/old/reme4_resource.md +++ b/docs4/old/reme4_resource.md @@ -44,7 +44,7 @@ hook任务： - auto-memory：在上下文满/每隔多少轮/session_end - - qwenpaw：直接传message session_id date，直接append session_{session_id}.jsonl @jinli + - qwenpaw：直接传message session_id date，直接append session_agent_{session_id}.jsonl @jinli - cc：给出新的path，对比我们保存的session_{session_id}.jsonl，看到增量msg，接着解析path的内容，保存到session_{session_id}.jsonl @sen - 注意保存的时候截断工具调用结构，防止太长 diff --git a/reme4/components/agent_wrapper/__init__.py b/reme4/components/agent_wrapper/__init__.py index 7e942e97..cd049bc6 100644 --- a/reme4/components/agent_wrapper/__init__.py +++ b/reme4/components/agent_wrapper/__init__.py @@ -4,8 +4,4 @@ from .base_agent_wrapper import BaseAgentWrapper from .as_agent_wrapper import AsAgentWrapper from .cc_agent_wrapper import CcAgentWrapper -__all__ = [ - "BaseAgentWrapper", - "AsAgentWrapper", - "CcAgentWrapper", -] +__all__ = ["BaseAgentWrapper", "AsAgentWrapper", "CcAgentWrapper"] diff --git a/reme4/components/agent_wrapper/as_agent_wrapper.py b/reme4/components/agent_wrapper/as_agent_wrapper.py index 3c29952e..fa85db04 100644 --- a/reme4/components/agent_wrapper/as_agent_wrapper.py +++ b/reme4/components/agent_wrapper/as_agent_wrapper.py @@ -1,14 +1,22 @@ """AgentScope backend for the unified agent wrapper.""" +import datetime +import uuid +import zoneinfo +from collections.abc import AsyncGenerator +from pathlib import Path from typing import Any, TYPE_CHECKING -from agentscope.agent import Agent -from agentscope.message import TextBlock, ToolResultState, UserMsg, SystemMsg +from agentscope.agent import Agent, ContextConfig, ModelConfig, ReActConfig +from agentscope.message import TextBlock, ToolResultState, UserMsg +from agentscope.permission import PermissionContext, PermissionMode +from agentscope.state import AgentState from agentscope.tool import FunctionTool, ToolChunk, Toolkit from .base_agent_wrapper import BaseAgentWrapper from ..as_llm import BaseAsLLM from ..component_registry import R +from ...utils import AsStateHandler if TYPE_CHECKING: from ..job.base_job import BaseJob @@ -16,14 +24,7 @@ if TYPE_CHECKING: @R.register("agentscope") class AsAgentWrapper(BaseAgentWrapper): - """Agent wrapper backed by AgentScope framework. - - Args: - as_llm: Name of the bound as_llm component (resolved via app_context). - Kwargs: - system_prompt: System prompt for the agent. - tools: list[BaseJob] to register as agent wrapper tools. - """ + """Agent wrapper backed by AgentScope framework.""" def __init__(self, as_llm: str = "default", **kwargs): super().__init__(**kwargs) @@ -33,17 +34,16 @@ class AsAgentWrapper(BaseAgentWrapper): def _make_tool(job: "BaseJob") -> FunctionTool: async def run_job(**kwargs) -> ToolChunk: response = await job(**kwargs) - return ToolChunk( - content=[TextBlock(text=str(response.answer))], - state=ToolResultState.SUCCESS if response.success else ToolResultState.ERROR, - ) + state = ToolResultState.SUCCESS if response.success else ToolResultState.ERROR + return ToolChunk(content=[TextBlock(text=str(response.answer))], state=state) tool = FunctionTool(func=run_job, name=job.name, description=job.description) if job.parameters: tool.input_schema = job.parameters return tool - async def reply(self, inputs: Any, session_id: str | None = None, **kwargs) -> tuple[str, Any]: + def _build_agent(self, inputs: Any, **kwargs) -> tuple[Agent, Any]: + """Build an Agent instance from kwargs. Returns (agent, processed_inputs).""" model = self.as_llm.model if self.as_llm else None if model is None: raise ValueError("AsAgentWrapper requires a bound as_llm component with a valid model.") @@ -51,34 +51,90 @@ class AsAgentWrapper(BaseAgentWrapper): for k, v in self.kwargs.items(): kwargs.setdefault(k, v) - output_schema: dict | None = kwargs.get("output_schema") - - tools: list["BaseJob"] = kwargs.get("tools", []) - toolkit = Toolkit(tools=[self._make_tool(job) for job in tools]) if tools else Toolkit() - system_prompt = kwargs.get("system_prompt", "You are a helpful assistant.") + tools: list["BaseJob"] = kwargs.get("tools", []) + toolkit = kwargs.get("toolkit") or ( + Toolkit(tools=[self._make_tool(job) for job in tools]) if tools else Toolkit() + ) + + perm_mode = PermissionMode(kwargs.get("permission_mode", "bypass")) + state = AgentState(permission_context=PermissionContext(mode=perm_mode)) agent = Agent( name=self.name, system_prompt=system_prompt, model=model, toolkit=toolkit, + state=state, + model_config=ModelConfig(**(kwargs.get("model_config") or {})), + context_config=ContextConfig(**(kwargs.get("context_config") or {})), + react_config=ReActConfig(**(kwargs.get("react_config") or {})), ) if isinstance(inputs, str): inputs = UserMsg(name="user", content=inputs) - if output_schema: - messages = [ - SystemMsg(name="system", content=system_prompt), - inputs, - ] - res = await model.generate_structured_output( - messages=messages, - structured_model=output_schema, - ) - return agent.state.session_id, res.content + return agent, inputs + + def _session_path(self, session_id: str) -> Path: + tz_name = self.app_context.app_config.timezone if self.app_context else None + if tz_name: + try: + dt = datetime.datetime.now(zoneinfo.ZoneInfo(tz_name)) + except Exception: + dt = datetime.datetime.now() + else: + dt = datetime.datetime.now() + date_str = dt.strftime("%Y-%m-%d") + resource = self.app_context.app_config.resource_dir if self.app_context else "resource" + return self.vault_path / resource / date_str / f"session_reme_{session_id}.jsonl" + + async def reply(self, inputs: Any, **kwargs) -> tuple[str, Any]: + session_id: str = kwargs.pop("session_id", "") + fork_session: bool = kwargs.pop("fork_session", False) + + agent, inputs = self._build_agent(inputs, **kwargs) + + if session_id: + path = self._session_path(session_id) + if path.exists(): + loaded = await AsStateHandler(path).load() + agent.state.session_id = loaded.session_id + agent.state.summary = loaded.summary + agent.state.context = loaded.context + agent.state.reply_id = loaded.reply_id + agent.state.cur_iter = loaded.cur_iter await agent.observe(inputs) await agent.reply() - return agent.state.session_id, agent.state.context[-1] + last_msg = agent.state.context[-1] + + if session_id: + if fork_session: + new_sid = uuid.uuid4().hex + agent.state.session_id = new_sid + save_path = self._session_path(new_sid) + else: + save_path = self._session_path(session_id) + save_path.parent.mkdir(parents=True, exist_ok=True) + await AsStateHandler(save_path).dump(agent.state) + + output_schema: dict | None = kwargs.get("output_schema") + if output_schema is not None: + assert self.as_llm is not None, "AsAgentWrapper requires a bound as_llm component with a valid model." + model = self.as_llm.model + assert model is not None, "AsAgentWrapper requires a bound as_llm component with a valid model." + res = await model.generate_structured_output( + messages=agent.state.context, + structured_model=output_schema, + ) + return agent.state.session_id, {"message": last_msg, "structured_output": res.content} + + return agent.state.session_id, last_msg + + async def reply_stream(self, inputs: Any, **kwargs) -> AsyncGenerator[Any, None]: + """Stream agent events via AgentScope's reply_stream API.""" + agent, inputs = self._build_agent(inputs, **kwargs) + + async for event in agent.reply_stream(inputs): + yield event diff --git a/reme4/components/agent_wrapper/base_agent_wrapper.py b/reme4/components/agent_wrapper/base_agent_wrapper.py index c3f83660..8c8f281b 100644 --- a/reme4/components/agent_wrapper/base_agent_wrapper.py +++ b/reme4/components/agent_wrapper/base_agent_wrapper.py @@ -1,8 +1,11 @@ """Base agent wrapper component.""" from abc import abstractmethod +from collections.abc import AsyncGenerator from typing import Any, TYPE_CHECKING +from pydantic import BaseModel + from ..base_component import BaseComponent from ...enumeration import ComponentEnum @@ -11,11 +14,7 @@ if TYPE_CHECKING: class BaseAgentWrapper(BaseComponent): - """Abstract base for agent wrapper components with swappable backends. - - Subclasses implement reply() which returns (session_id, last_message). - Supports fluent configuration via set_system_prompt() and add_tools(). - """ + """Abstract base for agent wrapper components with swappable backends.""" component_type = ComponentEnum.AGENT_WRAPPER @@ -29,11 +28,24 @@ class BaseAgentWrapper(BaseComponent): self.kwargs.setdefault("tools", []).extend(tools) return self - def set_output_schema(self, schema: dict) -> "BaseAgentWrapper": - """Set a JSON schema for structured output. Returns self for chaining.""" + def set_output_schema(self, schema: dict | type[BaseModel]) -> "BaseAgentWrapper": + """Set a JSON schema for structured output. Accepts dict or BaseModel class. Returns self for chaining.""" + if isinstance(schema, type) and issubclass(schema, BaseModel): + schema = schema.model_json_schema() self.kwargs["output_schema"] = schema return self + # TODO add skills + @abstractmethod - async def reply(self, inputs: Any, session_id: str | None = None, **kwargs) -> tuple[str, Any]: + async def reply(self, inputs: Any, **kwargs) -> tuple[str, Any]: """Send inputs to the agent and return (session_id, last_message).""" + + async def reply_stream(self, inputs: Any, **kwargs) -> AsyncGenerator[Any, None]: + """Stream agent events. Yields backend-specific event objects. + + Subclasses may override to provide streaming support. + Default implementation falls back to non-streaming reply and yields the final message. + """ + _, msg = await self.reply(inputs, **kwargs) + yield msg diff --git a/reme4/components/agent_wrapper/cc_agent_wrapper.py b/reme4/components/agent_wrapper/cc_agent_wrapper.py index d2d11d43..ba81d117 100644 --- a/reme4/components/agent_wrapper/cc_agent_wrapper.py +++ b/reme4/components/agent_wrapper/cc_agent_wrapper.py @@ -11,15 +11,7 @@ if TYPE_CHECKING: @R.register("claude_code") class CcAgentWrapper(BaseAgentWrapper): - """Agent wrapper backed by Claude Code SDK. - - Kwargs: - system_prompt: System prompt for the agent. - model: Claude model to use. - permission_mode: Permission mode for tool execution. - max_turns: Maximum conversation turns. - tools: list[BaseJob] to register as agent wrapper tools. - """ + """Agent wrapper backed by Claude Code SDK.""" @staticmethod def _make_tool(job: "BaseJob"): @@ -27,19 +19,11 @@ class CcAgentWrapper(BaseAgentWrapper): async def run_job(args): response = await job(**args) - return { - "content": [{"type": "text", "text": str(response.answer)}], - "is_error": not response.success, - } + return {"content": [{"type": "text", "text": str(response.answer)}], "is_error": not response.success} - return SdkMcpTool( - name=job.name, - description=job.description, - input_schema=job.parameters, - handler=run_job, - ) + return SdkMcpTool(name=job.name, description=job.description, input_schema=job.parameters, handler=run_job) - async def reply(self, inputs: Any, session_id: str | None = None, **kwargs) -> tuple[str, Any]: + async def reply(self, inputs: Any, **kwargs) -> tuple[str, Any]: from claude_agent_sdk import query, ResultMessage, create_sdk_mcp_server from claude_agent_sdk.types import ClaudeAgentOptions @@ -47,35 +31,26 @@ class CcAgentWrapper(BaseAgentWrapper): kwargs.setdefault(k, v) opts = ClaudeAgentOptions() - opts.system_prompt = kwargs.get("system_prompt", "You are a helpful assistant.") - opts.model = kwargs["model"] - opts.permission_mode = kwargs.get("permission_mode", None) - opts.max_turns = kwargs.get("max_turns", 50) - if session_id: - opts.session_id = session_id - opts.fork_session = True + skip_keys = {"tools", "output_schema"} + for k, v in kwargs.items(): + if k not in skip_keys and hasattr(opts, k): + setattr(opts, k, v) tools: list["BaseJob"] = kwargs.get("tools", []) if tools: sdk_tools = [self._make_tool(job) for job in tools] server = create_sdk_mcp_server(name="reme_tools", tools=sdk_tools) - if isinstance(opts.mcp_servers, dict): - opts.mcp_servers["reme"] = server - else: - opts.mcp_servers = {"reme": server} + opts.mcp_servers = (opts.mcp_servers if isinstance(opts.mcp_servers, dict) else {}) | {"reme": server} opts.allowed_tools.extend(job.name for job in tools) - output_schema = kwargs.get("output_schema") - if output_schema: + if output_schema := kwargs.get("output_schema"): opts.output_format = {"type": "json_schema", "schema": output_schema} - if isinstance(inputs, str): - prompt = inputs - else: + if not isinstance(inputs, str): raise NotImplementedError("Only string input is supported for Claude Code.") last_msg = None - async for msg in query(prompt=prompt, options=opts): + async for msg in query(prompt=inputs, options=opts): if isinstance(msg, ResultMessage): last_msg = msg diff --git a/reme4/components/file_chunker/base_file_chunker.py b/reme4/components/file_chunker/base_file_chunker.py index 53c59369..49312989 100644 --- a/reme4/components/file_chunker/base_file_chunker.py +++ b/reme4/components/file_chunker/base_file_chunker.py @@ -9,7 +9,7 @@ from ...schema import FileChunk, FileNode class BaseFileChunker(BaseComponent): - """Abstract base for file chunkers. Subclasses implement `parse`.""" + """Abstract base for file chunkers. Subclasses implement `chunk`.""" component_type = ComponentEnum.FILE_CHUNKER @@ -18,5 +18,5 @@ class BaseFileChunker(BaseComponent): self.supported_extensions: list[str] = supported_extensions or [] @abstractmethod - async def parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]: - """Parse a file into (node, chunks).""" + async def chunk(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]: + """Chunk a file into (node, chunks).""" diff --git a/reme4/components/file_chunker/default_file_chunker.py b/reme4/components/file_chunker/default_file_chunker.py index 2790ba60..a6513c42 100644 --- a/reme4/components/file_chunker/default_file_chunker.py +++ b/reme4/components/file_chunker/default_file_chunker.py @@ -37,7 +37,7 @@ class DefaultFileChunker(BaseFileChunker): front_matter = FileFrontMatter() return front_matter, text[end_idx + 4 :].lstrip("\n") - async def parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]: + async def chunk(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]: file_path = Path(path) stat = file_path.stat() rel_path = self.to_vault_relative(path) diff --git a/reme4/components/file_chunker/markdown_file_chunker.py b/reme4/components/file_chunker/markdown_file_chunker.py index 6439fbd6..fc996edc 100644 --- a/reme4/components/file_chunker/markdown_file_chunker.py +++ b/reme4/components/file_chunker/markdown_file_chunker.py @@ -122,7 +122,7 @@ class MarkdownFileChunker(BaseFileChunker): self.chunk_chars = max(100, chunk_chars) self.embed_toc = embed_toc - async def parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]: + async def chunk(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]: from mistletoe.markdown_renderer import MarkdownRenderer from mistletoe.block_token import Document diff --git a/reme4/components/job/base_job.py b/reme4/components/job/base_job.py index 3a414bee..b2c935ae 100644 --- a/reme4/components/job/base_job.py +++ b/reme4/components/job/base_job.py @@ -31,8 +31,6 @@ class BaseJob(BaseComponent): self.parameters = parameters or {} self.step_configs = steps or [] self.enable_serve = enable_serve - # Resolved at start: (cls, params) pairs. Steps are re-instantiated per call so they stay - # stateless across runs and concurrent invocations don't share mutable step state. self.step_specs: list[tuple[type["BaseStep"], dict]] = [] async def _start(self) -> None: diff --git a/reme4/config/default.yaml b/reme4/config/default.yaml index fab0b081..71c28356 100644 --- a/reme4/config/default.yaml +++ b/reme4/config/default.yaml @@ -1,17 +1,17 @@ vault_dir: .reme daily_dir: daily digest_dir: digest -resource_dir: "" +resource_dir: resource # language: zh service: backend: http jobs: - update_store_index_loop: + index_update_loop: backend: background - watch_paths: [ "daily", "digest" ] - suffix_filters: [ "md" ] + watch_dirs: [daily_dir, digest_dir, resource_dir] + watch_suffixes: [md, jsonl] steps: - backend: scan_store_changes_step recursive: true @@ -20,6 +20,34 @@ jobs: - backend: watch_changes_step dispatch_step: update_index_step + resource_watch_loop: + backend: background + watch_dirs: [resource_dir] + watch_suffixes: [md, txt, json, jsonl, csv, yaml, html] + dispatch_job: auto_resource + persist: true + steps: + - backend: scan_catalog_changes_step + recursive: true + - backend: update_catalog_step + persist: true + - backend: foreach_dispatch_step + - backend: watch_changes_step + dispatch_steps: [update_catalog_step, foreach_dispatch_step] + + digest_watch_loop: + backend: background + watch_dirs: [daily_dir, digest_dir] + watch_suffixes: [md] + persist: true + steps: + - backend: scan_catalog_changes_step + recursive: true + - backend: update_catalog_step + persist: true + - backend: watch_changes_step + dispatch_step: log_changes_step + version: backend: base description: "return reme package version" @@ -452,6 +480,24 @@ jobs: steps: - backend: auto_memory_step + auto_resource: + backend: base + description: "Auto-resource: interpret resource files into daily notes" + parameters: + type: object + properties: + file_path: + type: string + description: "vault-relative resource file path, e.g. resource/2026-06-06/report.pdf" + change: + type: string + description: "added/modified/deleted" + required: + - file_path + - change + steps: + - backend: auto_resource_step + components: tokenizer: default: @@ -488,11 +534,21 @@ components: default: backend: agentscope as_llm: default + permission_mode: bypass + react_config: + max_iters: 30 + context_config: + trigger_ratio: 0.8 + reserve_ratio: 0.1 + tool_result_limit: 3000 + model_config: + max_retries: 1 claude_code: backend: claude_code model: ${LLM_MODEL_NAME:-claude-opus-4-6} + system_prompt: "You are a helpful assistant." permission_mode: bypassPermissions - max_turns: 10 + max_turns: 50 file_graph: default: @@ -508,7 +564,7 @@ components: supported_extensions: [ "md" ] default: backend: default - supported_extensions: [ "txt", "html", "json", "jsonl", "yaml", "py" ] + supported_extensions: [ "jsonl" ] keyword_index: default: diff --git a/reme4/pyproject.toml b/reme4/pyproject.toml index abba8874..13243b46 100644 --- a/reme4/pyproject.toml +++ b/reme4/pyproject.toml @@ -61,7 +61,7 @@ Documentation = "https://reme.agentscope.io/" Repository = "https://github.com/agentscope-ai/ReMe" [project.scripts] -reme4 = "reme4.reme:main" +reme = "reme4.reme:main" [tool.setuptools] packages = { find = { where = [".."], include = ["reme4*"] } } diff --git a/reme4/schema/application_config.py b/reme4/schema/application_config.py index 25e6e8b3..5875dcb6 100644 --- a/reme4/schema/application_config.py +++ b/reme4/schema/application_config.py @@ -34,7 +34,7 @@ class ApplicationConfig(BaseModel): daily_dir: str = Field(default="daily", description="Subdirectory for daily memory") digest_dir: str = Field(default="digest", description="Subdirectory for digest memory") enable_logo: bool = Field(default=True, description="Show ASCII logo on startup") - timezone: str | None = Field(default=None, description="IANA timezone (e.g. 'Asia/Shanghai'); None uses local time") + timezone: str | None = Field(default="Asia/Shanghai", description="IANA timezone; None uses local time") language: str = Field(default="", description="Default language for LLM interactions") log_to_console: bool = Field(default=True, description="Log to console") log_to_file: bool = Field(default=True, description="Log to file") diff --git a/reme4/steps/__init__.py b/reme4/steps/__init__.py index fc492938..30e67b5d 100644 --- a/reme4/steps/__init__.py +++ b/reme4/steps/__init__.py @@ -9,6 +9,7 @@ from .common.stream_demo import StreamDemoStep1, StreamDemoStep2 from .common.version import VersionStep from .evolve.auto_dream import AutoDreamStep from .evolve.auto_memory import AutoMemoryStep +from .evolve.auto_resource import AutoResourceStep from .evolve.dream import DreamStep from .file_io.daily_create import DailyCreateStep from .file_io.daily_list import DailyListStep @@ -24,8 +25,8 @@ from .file_io.read import ReadStep from .file_io.read_image import ReadImageStep from .file_io.stat import StatStep from .file_io.write import WriteStep -from .index.channel_notify import ChannelNotifyStep -from .index.claim_channel import ClaimChannelStep +from .channel.channel_notify import ChannelNotifyStep +from .channel.claim_channel import ClaimChannelStep from .index.clear_and_scan import ClearAndScanStep from .index.node_search import NodeSearchStep from .index.scan_changes import ScanCatalogChangesStep, ScanStoreChangesStep @@ -33,6 +34,8 @@ from .index.search import SearchStep from .index.traverse import TraverseStep from .index.update_catalog import UpdateCatalogStep from .index.update_index import UpdateIndexStep +from .index.foreach_dispatch import ForeachDispatchStep +from .index.log_changes import LogChangesStep from .index.watch_changes import WatchChangesStep from .transfer.download import DownloadStep from .transfer.ingest import IngestStep @@ -51,6 +54,7 @@ __all__ = [ "VersionStep", # evolve "AutoMemoryStep", + "AutoResourceStep", # file_io "DeleteStep", "EditStep", @@ -68,10 +72,13 @@ __all__ = [ "FrontmatterDeleteStep", "FrontmatterReadStep", "FrontmatterUpdateStep", - # index + # channel "ChannelNotifyStep", "ClaimChannelStep", + # index "ClearAndScanStep", + "ForeachDispatchStep", + "LogChangesStep", "NodeSearchStep", "ScanCatalogChangesStep", "ScanStoreChangesStep", diff --git a/reme4/steps/base_step.py b/reme4/steps/base_step.py index beeeb443..4eb7854d 100644 --- a/reme4/steps/base_step.py +++ b/reme4/steps/base_step.py @@ -2,20 +2,18 @@ import copy from abc import abstractmethod, ABC -from pathlib import Path from typing import TypeVar, TYPE_CHECKING -from agentscope.message import TextBlock from agentscope.model import ChatModelBase -from agentscope.tool import Toolkit, FunctionTool, ToolChunk +from ..components.agent_wrapper.base_agent_wrapper import BaseAgentWrapper from ..components.base_component import ComponentMixin -from ..components.file_chunker import BaseFileChunker +from ..components.file_catalog import BaseFileCatalog from ..components.file_store import BaseFileStore from ..components.prompt_handler import PromptHandler from ..components.runtime_context import RuntimeContext from ..enumeration import ComponentEnum -from ..schema import FileChunk, FileNode, Response +from ..schema import Response if TYPE_CHECKING: from ..components import ApplicationContext @@ -103,6 +101,8 @@ class BaseStep(ComponentMixin, ABC): component_type = ComponentEnum.STEP as_llm: ChatModelBase = Ref(ChatModelBase, ComponentEnum.AS_LLM, "model") + agent_wrapper: BaseAgentWrapper = Ref(BaseAgentWrapper, ComponentEnum.AGENT_WRAPPER, optional=True) + file_catalog: BaseFileCatalog = Ref(BaseFileCatalog, ComponentEnum.FILE_CATALOG, optional=True) file_store: BaseFileStore = Ref(BaseFileStore, ComponentEnum.FILE_STORE) def __new__(cls, *args, **kwargs): @@ -157,36 +157,6 @@ class BaseStep(ComponentMixin, ABC): self.context.apply_mapping(self.output_mapping) return result - async def parse_file(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]: - """Parse ``path`` with the parser whose ``supported_extensions`` claims its suffix. - - First registered match wins (config insertion order). Falls back to the - ``default`` parser (stat-only) when no parser claims the suffix — that's - how attachments / binaries / unknown types still produce a FileNode. - """ - if self.app_context is None: - raise RuntimeError("app_context is not set when resolving file chunker") - file_chunker_dict: dict[str, BaseFileChunker] = self.app_context.components[ComponentEnum.FILE_CHUNKER] - - suffix = Path(path).suffix.lstrip(".").lower() - - parser: BaseFileChunker | None = None - if suffix: - for candidate in file_chunker_dict.values(): - if suffix in {ext.lower().lstrip(".") for ext in candidate.supported_extensions}: - parser = candidate - break - - if parser is None: - parser = file_chunker_dict.get("default") - - if parser is None: - raise RuntimeError( - f"No file chunker supports {path} (suffix={suffix!r}) and no 'default' chunker is configured", - ) - - return await parser.parse(path) - def prompt_format(self, prompt_name: str, **kwargs) -> str: """Format a named prompt template with the given kwargs.""" return self.prompt.prompt_format(prompt_name=prompt_name, **kwargs) @@ -211,25 +181,3 @@ class BaseStep(ComponentMixin, ABC): if job is None: raise RuntimeError(f"Job {name} not found") return await job(**kwargs) - - def add_as_tool(self, toolkit: Toolkit, job_name: str, **kwargs) -> None: - """Add the step as a tool to the toolkit.""" - job: "BaseJob | None" = self.get_job(job_name) - if job is None: - raise RuntimeError(f"Job {job_name} not found") - - async def run_job(**_kwargs) -> ToolChunk: - response = await job(**{**_kwargs, **kwargs}) - return ToolChunk( - content=[TextBlock(text=str(response.answer))], - state="success" if response.success else "error", - ) - - tool = FunctionTool( - func=run_job, - name=job_name, - description=job.description, - ) - if job.parameters: - tool.input_schema = job.parameters - toolkit.tool_groups[0].tools.append(tool) diff --git a/reme4/steps/channel/__init__.py b/reme4/steps/channel/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/reme4/steps/index/channel_notify.py b/reme4/steps/channel/channel_notify.py similarity index 100% rename from reme4/steps/index/channel_notify.py rename to reme4/steps/channel/channel_notify.py diff --git a/reme4/steps/index/claim_channel.py b/reme4/steps/channel/claim_channel.py similarity index 100% rename from reme4/steps/index/claim_channel.py rename to reme4/steps/channel/claim_channel.py diff --git a/reme4/steps/common/llm_demo.py b/reme4/steps/common/llm_demo.py index 55bf419f..4dd0ac9a 100644 --- a/reme4/steps/common/llm_demo.py +++ b/reme4/steps/common/llm_demo.py @@ -1,11 +1,7 @@ -"""Demo step that drives an Agent via BaseStep.as_llm.""" +"""Demo step that drives an Agent via the agent_wrapper component.""" from typing import Type -from agentscope.agent import Agent -from agentscope.state import AgentState -from agentscope.message import Msg, TextBlock -from agentscope.permission import PermissionContext, PermissionMode from agentscope.tool import FunctionTool, Toolkit from pydantic import BaseModel @@ -25,7 +21,7 @@ def add(a: float, b: float) -> str: @R.register("llm_demo_step") class LLMDemoStep(BaseStep): - """Drive an Agent powered by ``self.as_llm``. + """Drive an Agent powered by the ``agent_wrapper`` component. Inputs (from RuntimeContext): query (str, required): user message content. @@ -36,7 +32,7 @@ class LLMDemoStep(BaseStep): The agent's final reply text. """ - DEFAULT_SYS_PROMPT = "You are a concise assistant. Reply in one short sentence." + DEFAULT_SYS_PROMPT = "You are a helpful assistant. Provide clear and detailed responses." async def execute(self): assert self.context is not None @@ -52,31 +48,24 @@ class LLMDemoStep(BaseStep): toolkit = Toolkit(tools=[FunctionTool(add)]) if use_add_tool else Toolkit() - agent = Agent( - name=self.name, - system_prompt=sys_prompt, - model=self.as_llm, - toolkit=toolkit, - state=AgentState( - permission_context=PermissionContext( - mode=PermissionMode.BYPASS, - ), - ), - ) + wrapper_kwargs = { + "system_prompt": sys_prompt, + "toolkit": toolkit, + } + if structured_model is not None: + wrapper_kwargs["output_schema"] = structured_model - response: Msg = await agent.reply( - Msg(name="user", role="user", content=[TextBlock(text=query)]), - ) - text = (response.get_text_content() or "").strip() - self.logger.info(f"[{self.name}] response: {text!r}") + _, result = await self.agent_wrapper.reply(query, **wrapper_kwargs) structured_content: dict | None = None - if structured_model is not None: - structured_resp = await self.as_llm.generate_structured_output( - agent.state.context, - structured_model=structured_model, - ) - structured_content = structured_resp.content + if isinstance(result, dict) and "message" in result: + msg = result["message"] + structured_content = result["structured_output"] + else: + msg = result + + text = (msg.get_text_content() or "").strip() + self.logger.info(f"[{self.name}] response: {text!r}") self.context.response.success = True self.context.response.answer = text diff --git a/reme4/steps/common/stream_llm_demo.py b/reme4/steps/common/stream_llm_demo.py index 7513840b..4eb8ca53 100644 --- a/reme4/steps/common/stream_llm_demo.py +++ b/reme4/steps/common/stream_llm_demo.py @@ -1,8 +1,7 @@ -"""Demo step that drives an Agent via BaseStep.as_llm with streaming output.""" +"""Demo step that drives an Agent via the agent_wrapper component with streaming output.""" import json -from agentscope.agent import Agent from agentscope.event import ( TextBlockDeltaEvent, ThinkingBlockDeltaEvent, @@ -12,9 +11,7 @@ from agentscope.event import ( ModelCallEndEvent, ReplyStartEvent, ) -from agentscope.message import Msg, TextBlock -from agentscope.permission import PermissionContext, PermissionMode -from agentscope.state import AgentState +from agentscope.message import Msg from agentscope.tool import FunctionTool, Toolkit from ..base_step import BaseStep @@ -34,11 +31,11 @@ def add(a: float, b: float) -> str: @R.register("stream_llm_demo_step") class StreamLLMDemoStep(BaseStep): - """Drive an Agent powered by ``self.as_llm`` with streaming output. + """Drive an Agent powered by the ``agent_wrapper`` component with streaming output. When streaming is enabled on the context, text/thinking/tool events are pushed chunk-by-chunk via ``self.context.add_stream_string``. - When streaming is not enabled, falls back to non-streaming ``agent.reply``. + When streaming is not enabled, falls back to non-streaming reply. Inputs (from RuntimeContext): query (str, required): user message content. @@ -49,7 +46,7 @@ class StreamLLMDemoStep(BaseStep): The agent's final reply text. """ - DEFAULT_SYS_PROMPT = "You are a concise assistant. Reply in one short sentence." + DEFAULT_SYS_PROMPT = "You are a helpful assistant. Provide clear and detailed responses." async def execute(self): assert self.context is not None @@ -64,27 +61,18 @@ class StreamLLMDemoStep(BaseStep): toolkit = Toolkit(tools=[FunctionTool(add)]) if use_add_tool else Toolkit() - agent = Agent( - name=self.name, - system_prompt=sys_prompt, - model=self.as_llm, - toolkit=toolkit, - state=AgentState( - permission_context=PermissionContext( - mode=PermissionMode.BYPASS, - ), - ), - ) - - input_msg = Msg(name="user", role="user", content=[TextBlock(text=query)]) + wrapper_kwargs = { + "system_prompt": sys_prompt, + "toolkit": toolkit, + } if self.context.stream: - text = await self._stream_reply(agent, input_msg) + text = await self._stream_reply(query, **wrapper_kwargs) else: - response: Msg = await agent.reply(input_msg) - text = (response.get_text_content() or "").strip() + _, msg = await self.agent_wrapper.reply(query, **wrapper_kwargs) + text = (msg.get_text_content() or "").strip() - self.logger.info(f"[{self.name}] response: {text!r}") + self.logger.debug(f"[{self.name}] response: {text!r}") self.context.response.success = True self.context.response.answer = text @@ -98,12 +86,12 @@ class StreamLLMDemoStep(BaseStep): ) return self.context.response - async def _stream_reply(self, agent: Agent, input_msg: Msg) -> str: + async def _stream_reply(self, query: str, **wrapper_kwargs) -> str: """Stream agent reply events to the context stream queue.""" assert self.context is not None reply_msg: Msg | None = None - async for event in agent.reply_stream(input_msg): + async for event in self.agent_wrapper.reply_stream(query, **wrapper_kwargs): if isinstance(event, ReplyStartEvent): reply_msg = Msg( id=event.reply_id, diff --git a/reme4/steps/evolve/auto_dream.py b/reme4/steps/evolve/auto_dream.py index d2efd6e3..544d783b 100644 --- a/reme4/steps/evolve/auto_dream.py +++ b/reme4/steps/evolve/auto_dream.py @@ -49,11 +49,9 @@ from pathlib import Path from pydantic import BaseModel, Field +from ._evolve import now from .dream import DreamStep, DreamResult -from ..base_step import Ref from ...components import R -from ...components.file_catalog import BaseFileCatalog -from ...enumeration import ComponentEnum from ...schema import FileNode @@ -76,8 +74,6 @@ class AutoDreamStep(DreamStep): """Scan ``daily/.md`` + ``daily//`` and dream each file whose ``st_mtime`` doesn't already match its ``file_catalog`` entry.""" - file_catalog: BaseFileCatalog = Ref(BaseFileCatalog, ComponentEnum.FILE_CATALOG) - def __init__(self, persist: bool = True, **kwargs): super().__init__(**kwargs) self.persist: bool = persist @@ -92,7 +88,8 @@ class AutoDreamStep(DreamStep): cfg = self.app_context.app_config if self.app_context is not None else None daily_dir = (cfg.daily_dir if cfg else "") or "daily" - today = date_input or self._now().strftime("%Y-%m-%d") + tz = self.app_context.app_config.timezone if self.app_context is not None else None + today = date_input or now(tz).strftime("%Y-%m-%d") vault = self._vault_dir() files = _scan_today_files(vault, today, daily_dir) diff --git a/reme4/steps/evolve/auto_memory.py b/reme4/steps/evolve/auto_memory.py index 3b0b4800..f3d4b5e6 100644 --- a/reme4/steps/evolve/auto_memory.py +++ b/reme4/steps/evolve/auto_memory.py @@ -1,42 +1,68 @@ -"""``auto_memory`` — record conversation facts into a daily note. +"""auto_memory — record conversation facts into a daily note via an agent.""" -Calls ``daily_create`` as a system call to provision the note path, -then hands off to a ReAct agent that reads existing content (if any), -decides what to preserve, and writes the note via ``read`` / ``edit`` -/ ``frontmatter_update`` / ``write`` tools. +from pathlib import Path -Inputs (from RuntimeContext): - messages (list[Msg], optional): conversation slice to inspect. - Mutually exclusive with ``transcript_path``; if both are - provided, ``messages`` wins. - transcript_path (str, optional): absolute path to a Claude Code - transcript JSONL file. When provided (and ``messages`` is - empty), the step parses the file via - :func:`reme4.utils.transcript.load_messages_from_transcript` - and proceeds as if those were the messages. This is what the - ``reme-service`` plugin's PreCompact / SessionEnd hooks pass - in directly via ``type: mcp_tool``, replacing the temporary - spawn-subagent bridge. - session_id (str, optional): passed to daily_create to determine - the note path. - memory_hint (str, optional): caller-supplied hint for the agent. - timezone (str, optional): IANA timezone for date resolution. - -Output (written to context.response): - answer: one-line summary from the agent. - metadata: {path, created, n_messages, transcript_path?}. -""" - -from agentscope.agent import Agent -from agentscope.message import Msg, TextBlock -from agentscope.permission import PermissionContext, PermissionMode -from agentscope.state import AgentState -from agentscope.tool import Toolkit +import aiofiles +from agentscope.message import Msg from ._evolve import format_history, now from ..base_step import BaseStep from ...components import R -from ...utils.transcript import load_messages_from_transcript + +_TOOL_OUTPUT_MAX = 2048 +_TOOL_OUTPUT_HALF = 1024 + + +def _truncate_text(text: str) -> str: + if len(text) <= _TOOL_OUTPUT_MAX: + return text + return text[:_TOOL_OUTPUT_HALF] + "\n...(truncated)...\n" + text[-_TOOL_OUTPUT_HALF:] + + +def _sanitize_tool_result(block): + output = block.output + if isinstance(output, str): + truncated = _truncate_text(output) + if truncated is output: + return block + return block.model_copy(update={"output": truncated}) + new_output = [] + changed = False + for item in output: + if item.type == "data": + changed = True + continue + if item.type == "text": + truncated = _truncate_text(item.text) + if truncated is not item.text: + changed = True + new_output.append(item.model_copy(update={"text": truncated})) + else: + new_output.append(item) + else: + new_output.append(item) + if not changed: + return block + return block.model_copy(update={"output": new_output}) + + +def _sanitize_msg_for_save(msg: Msg) -> Msg: + new_content = [] + changed = False + for block in msg.content: + if block.type == "data" and hasattr(block, "source") and getattr(block.source, "type", None) == "base64": + changed = True + continue + if block.type == "tool_result": + sanitized = _sanitize_tool_result(block) + if sanitized is not block: + changed = True + new_content.append(sanitized) + else: + new_content.append(block) + if not changed: + return msg + return msg.model_copy(update={"content": new_content}) @R.register("auto_memory_step") @@ -47,6 +73,54 @@ class AutoMemoryStep(BaseStep): super().__init__(**kwargs) self.agent_tools: list[str] = ["read", "edit", "frontmatter_update", "write"] + def _session_path(self, session_id: str, tz: str | None) -> Path: + current = now(tz) + date_str = current.strftime("%Y-%m-%d") + resource = self.app_context.app_config.resource_dir if self.app_context else "resource" + return self.file_store.vault_path / resource / date_str / f"session_agent_{session_id}.jsonl" + + async def _save_session_messages(self, session_id: str, messages: list[Msg], tz: str | None) -> None: + if not session_id or not messages: + return + + path = self._session_path(session_id, tz) + + existing: list[Msg] = [] + if path.exists(): + async with aiofiles.open(path, encoding="utf-8") as f: + content = await f.read() + for line in content.splitlines(): + line = line.strip() + if line: + try: + existing.append(Msg.model_validate_json(line)) + except Exception: + pass + + by_id: dict[str, Msg] = {} + for msg in existing: + by_id[msg.id] = msg + for msg in messages: + by_id[msg.id] = msg + merged = sorted(by_id.values(), key=lambda m: m.created_at) + + can_append = 0 < len(existing) <= len(merged) and all( + merged[i].id == existing[i].id for i in range(len(existing)) + ) + + path.parent.mkdir(parents=True, exist_ok=True) + + if can_append: + new_msgs = merged[len(existing) :] + if new_msgs: + async with aiofiles.open(path, "a", encoding="utf-8") as f: + for msg in new_msgs: + await f.write(_sanitize_msg_for_save(msg).model_dump_json() + "\n") + else: + async with aiofiles.open(path, "w", encoding="utf-8") as f: + for msg in merged: + await f.write(_sanitize_msg_for_save(msg).model_dump_json() + "\n") + @staticmethod def _to_msg(item) -> Msg: if isinstance(item, Msg): @@ -58,34 +132,20 @@ class AutoMemoryStep(BaseStep): async def execute(self): assert self.context is not None raw_messages = self.context.get("messages") or [] - transcript_path: str = self.context.get("transcript_path", "") or "" session_id: str = self.context.get("session_id", "") memory_hint: str = self.context.get("memory_hint", "") - current = now(self.context.get("timezone")) - - # If caller passed transcript_path (the canonical Claude Code hook - # input) instead of messages, parse it here so the rest of the step - # stays unchanged. - if not raw_messages and transcript_path: - raw_messages = load_messages_from_transcript(transcript_path) - self.logger.info( - f"[{self.name}] loaded {len(raw_messages)} messages from transcript_path={transcript_path}", - ) + tz = self.app_context.app_config.timezone if self.app_context is not None else None + current = now(tz) messages: list[Msg] = [self._to_msg(item) for item in raw_messages] + await self._save_session_messages(session_id, messages, tz) + if not messages: self.context.response.success = True - reason = ( - f"Skipped: no messages in transcript_path={transcript_path}" - if transcript_path - else "Skipped: no messages supplied" - ) - self.context.response.answer = reason - self.context.response.metadata.update( - {"n_messages": 0, "transcript_path": transcript_path}, - ) - self.logger.info(f"[{self.name}] skipped: {reason} session_id={session_id!r}") + self.context.response.answer = "Skipped: no messages" + self.context.response.metadata.update({"n_messages": 0}) + self.logger.info(f"[{self.name}] Skipped: no messages session_id={session_id!r}") return create_response = await self.run_job("daily_create", session_id=session_id) @@ -97,29 +157,10 @@ class AutoMemoryStep(BaseStep): note_path: str = create_response.metadata["path"] created: bool = create_response.metadata["created"] - self.logger.info( - f"[{self.name}] note_path={note_path} created={created} " - f"messages={len(messages)} hint={'yes' if memory_hint else 'no'}", - ) - - toolkit = Toolkit() - for job_name in self.agent_tools: - self.add_as_tool(toolkit, job_name) - - agent = Agent( - name="auto_memory", - model=self.as_llm, - system_prompt=self.prompt_format("system_prompt"), - toolkit=toolkit, - state=AgentState( - permission_context=PermissionContext( - mode=PermissionMode.BYPASS, - ), - ), - ) + self.logger.info(f"[{self.name}] {note_path} created={created} msgs={len(messages)} hint={bool(memory_hint)}") template_key = "user_message_create" if created else "user_message_update" - user_message: str = self.prompt_format( + user_message = self.prompt_format( template_key, today=current.strftime("%Y-%m-%d"), vault_dir=str(self.file_store.vault_path), @@ -128,16 +169,16 @@ class AutoMemoryStep(BaseStep): history=format_history(messages), ) - final_msg: Msg = await agent.reply(Msg(name="reme", role="user", content=[TextBlock(text=user_message)])) + tools = [self.get_job(name) for name in self.agent_tools] + _, msg = await self.agent_wrapper.reply( + user_message, + system_prompt=self.prompt_format("system_prompt"), + tools=tools, + ) self.context.response.success = True - self.context.response.answer = (final_msg.get_text_content() or "").strip() + self.context.response.answer = (msg.get_text_content() or "").strip() self.context.response.metadata.update( - { - "path": note_path, - "created": created, - "n_messages": len(messages), - "transcript_path": transcript_path, - }, + {"path": note_path, "created": created, "n_messages": len(messages)}, ) - self.logger.info(f"[{self.name}] done note_path={note_path}") + self.logger.info(f"[{self.name}] done {note_path}") diff --git a/reme4/steps/evolve/auto_resource.py b/reme4/steps/evolve/auto_resource.py new file mode 100644 index 00000000..5f02d86f --- /dev/null +++ b/reme4/steps/evolve/auto_resource.py @@ -0,0 +1,144 @@ +"""auto_resource — interpret resource files into daily notes via an agent.""" + +import hashlib +from pathlib import PurePosixPath + +import aiofiles +from watchfiles import Change + +from ..base_step import BaseStep +from ...components import R + + +def _compute_session_id(filename: str) -> str: + """Return 'resource_' + first 8 hex chars of MD5(filename).""" + digest = hashlib.md5(filename.encode()).hexdigest()[:8] + return f"resource_{digest}" + + +def _parse_resource_path(file_path: str, resource_dir: str) -> tuple[str, str]: + """Extract (date, filename) from a resource path like 'resource/2026-06-06/report.pdf'. + + Returns (date_str, filename) where filename may contain subdirectories. + """ + parts = PurePosixPath(file_path).parts + # Strip leading resource_dir prefix + prefix_parts = PurePosixPath(resource_dir).parts + if parts[: len(prefix_parts)] == prefix_parts: + parts = parts[len(prefix_parts) :] + # First segment is date, rest is filename + date_str = parts[0] if parts else "" + filename = str(PurePosixPath(*parts[1:])) if len(parts) > 1 else "" + return date_str, filename + + +@R.register("auto_resource_step") +class AutoResourceStep(BaseStep): + """Interpret resource files into daily notes via an Agent.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.agent_tools: list[str] = ["read", "edit", "frontmatter_update", "write"] + + def _normalize_change(self, raw) -> Change | None: + if isinstance(raw, Change): + return raw + if isinstance(raw, str): + return Change.__members__.get(raw) + return None + + async def _handle_delete(self, date_str: str, session_id: str) -> None: + daily_dir = self.app_context.app_config.daily_dir if self.app_context else "daily" + note_rel = f"{daily_dir}/{date_str}/session_agent_{session_id}.md" + note_abs = self.vault_path / note_rel + + if note_abs.is_file(): + note_abs.unlink() + self.logger.info(f"[{self.name}] Deleted file: {note_rel}") + + await self.file_store.delete([note_rel]) + + self.context.response.success = True + self.context.response.answer = f"Deleted resource note: {note_rel}" + self.context.response.metadata.update({"path": note_rel, "action": "deleted"}) + + async def _handle_upsert(self, file_path: str, date_str: str, session_id: str, created: bool) -> None: + create_response = await self.run_job("daily_create", session_id=session_id, date=date_str) + if not create_response.success: + self.context.response.success = False + self.context.response.answer = f"daily_create failed: {create_response.answer}" + return + + note_path: str = create_response.metadata["path"] + note_created: bool = create_response.metadata["created"] + + # Read resource file content + abs_path = self.vault_path / file_path + if not abs_path.is_file(): + self.context.response.success = False + self.context.response.answer = f"Resource file not found: {file_path}" + return + + async with aiofiles.open(abs_path, encoding="utf-8", errors="replace") as f: + file_content = await f.read() + + template_key = "user_message_create" if created else "user_message_update" + user_message = self.prompt_format( + template_key, + vault_dir=str(self.vault_path), + note_path=note_path, + file_path=file_path, + file_content=file_content, + date=date_str, + ) + + tools = [self.get_job(name) for name in self.agent_tools] + _, msg = await self.agent_wrapper.reply( + user_message, + system_prompt=self.prompt_format("system_prompt"), + tools=tools, + ) + + self.context.response.success = True + self.context.response.answer = (msg.get_text_content() or "").strip() + self.context.response.metadata.update( + {"path": note_path, "created": note_created, "action": "added" if created else "modified"}, + ) + self.logger.info(f"[{self.name}] done {note_path}") + + async def execute(self): + assert self.context is not None + file_path: str = self.context.get("file_path", "") + raw_change = self.context.get("change", "") + + if not file_path: + self.context.response.success = False + self.context.response.answer = "Missing file_path" + return + + change = self._normalize_change(raw_change) + if change is None: + self.context.response.success = False + self.context.response.answer = f"Invalid change type: {raw_change}" + return + + resource_dir = self.app_context.app_config.resource_dir if self.app_context else "resource" + date_str, filename = _parse_resource_path(file_path, resource_dir) + + if not date_str or not filename: + self.context.response.success = False + self.context.response.answer = f"Cannot parse date/filename from: {file_path}" + return + + session_id = _compute_session_id(filename) + self.logger.info(f"[{self.name}] {change.name} file_path={file_path} session_id={session_id}") + + if change == Change.deleted: + await self._handle_delete(date_str, session_id) + else: + await self._handle_upsert( + file_path, + date_str, + session_id, + created=change == Change.added, + ) diff --git a/reme4/steps/evolve/auto_resource.yaml b/reme4/steps/evolve/auto_resource.yaml new file mode 100644 index 00000000..721750e8 --- /dev/null +++ b/reme4/steps/evolve/auto_resource.yaml @@ -0,0 +1,218 @@ +system_prompt: | + You are an automatic resource interpretation system. Your job is to read a resource file and record a structured summary into a daily note at the specified path. Think about what information in this file would be most useful for future retrieval and understanding. + + ## What to Record + + - **Core content**: the main information, data, or knowledge in the file + - **Structure**: how the file is organized (sections, chapters, tables, etc.) + - **Key details**: important numbers, names, dates, decisions, or conclusions + - **Context**: what this file is about, its purpose, and how it relates to other work + - **Actionable items**: any tasks, deadlines, or follow-ups mentioned + + Be comprehensive — every significant fact should appear. Quote original wording or numbers verbatim at key points. + + ## Body Format + + Free-form — use whatever structure best fits the content (headings, lists, tables, etc.). The only hard rule is **completeness** and **faithfulness** to the source. + + ## Frontmatter Rules + + - `name` = the filename stem, copied verbatim. Do not Title-Case or rewrite it. + - `description` = a thorough summary; vague descriptions like "notes" / "misc" are unacceptable. + - **Never set `status`** — it is a field reserved for downstream processing. +system_prompt_zh: | + 你是自动资源解读系统。你的职责是读取一个资源文件，并将结构化摘要记录到指定路径的日记中。思考这个文件中哪些信息对未来检索和理解最有价值。 + + ## 记录什么 + + - **核心内容**：文件中的主要信息、数据或知识 + - **结构**：文件如何组织（章节、表格等） + - **关键细节**：重要的数字、名称、日期、决策或结论 + - **上下文**：这个文件关于什么、它的目的、以及与其他工作的关联 + - **可操作项**：提到的任何任务、截止日期或后续跟进 + + 要全面——每一条重要事实都应出现。关键处逐字引用原始措辞或数字。 + + ## 正文格式 + + 自由格式——用最适合内容的结构（标题、列表、表格等）。唯一的硬性规则是**完整性**和对原文的**忠实性**。 + + ## Frontmatter 规则 + + - `name` = 文件名 stem，逐字照抄。不要 Title-Case 化，不要改写。 + - `description` = 详细总结；模糊的描述如 "notes" / "misc" 不可接受。 + - **永远不要设置 `status`**——它是下游处理保留的字段。 + +user_message_create: | + Date: {date} + Vault directory: {vault_dir} + Resource file: {file_path} + Target note path: {note_path} + + # Resource File Content + + {file_content} + + # Your Task + + Interpret the resource file above and write a structured summary into the target note. + + ## Step 1 — Skip Check + + Does the resource file contain substantive information worth recording? If the file is empty, corrupted, or contains no meaningful content → reply with a brief skip message and stop (do not call any tools). + + When truly ambiguous, default to writing — losing information is worse than writing one extra note. + + ## Step 2 — Write + + The target file is a newly created empty file. Write the full content in one shot: + `write path={note_path} name= description= content=` + + - `name` must equal the filename stem of the target path (the part between the last `/` and `.md`), copied verbatim. + - `description` must be a thorough summary of the body — specific enough that the description alone conveys all key information. + + ## Step 3 — Summary + + State in one sentence what you did (which file was created). This is your final text output. + + ## Boundaries + + - Only operate on one target path: `{note_path}`. Do not touch other notes. +user_message_create_zh: | + 日期：{date} + Vault 目录：{vault_dir} + 资源文件：{file_path} + 目标笔记路径：{note_path} + + # 资源文件内容 + + {file_content} + + # 你的任务 + + 解读上述资源文件，将结构化摘要写入目标笔记。 + + ## 步骤 1 — 跳过检查 + + 资源文件是否包含值得记录的实质性信息？如果文件为空、损坏或没有有意义的内容 → 回复一条简短的跳过消息并停止（不调用任何工具）。 + + 当真正模棱两可时，默认写入——丢失信息比多写一条笔记更糟。 + + ## 步骤 2 — 写入 + + 目标文件是新建的空文件。一次性写入完整内容： + `write path={note_path} name= description= content=<正文>` + + - `name` 必须等于目标路径的文件名 stem（最后一个 `/` 与 `.md` 之间的部分），逐字照抄。 + - `description` 必须是正文的详尽总结——具体到仅凭 description 就能传达全部核心信息。 + + ## 步骤 3 — 总结 + + 用一句话说明你做了什么（创建了哪个文件）。这是你最后一次文本输出。 + + ## 边界 + + - 只针对一个目标路径：`{note_path}`。不要碰其他笔记。 + +user_message_update: | + Date: {date} + Vault directory: {vault_dir} + Resource file: {file_path} + Target note path: {note_path} + + # Resource File Content (Updated) + + {file_content} + + # Your Task + + The resource file has been updated. Re-interpret it and update the existing note at the target path. + + ## Step 1 — Read Existing Content + + Call `read path={note_path}` to inspect the current note content. + - If the body is empty (only frontmatter, no actual content) → treat as new, jump to **Step 2b**. + - If there is body content → go to **Step 2a** to merge. + + ## Step 2a — Merge Update + + The note already has content from a previous version of the resource file. Your task is to update it to reflect the current version. + + Update rules: + - **Removed content**: delete sections that no longer exist in the resource file. + - **New content**: add sections for newly added information. + - **Modified content**: rewrite affected sections to match the current file. + - **Unchanged content**: leave as-is. + + Execution: + 1. Use `edit path={note_path} old= new=` for each section that needs updating. You may call `edit` multiple times. + 2. After body changes, refresh the frontmatter description: `frontmatter_update path={note_path} metadata={{"description": ""}}`. + 3. If `edit` fails repeatedly (e.g., cannot find the original text), fall back to `write path={note_path} name= description= content=` for a complete rewrite. + + ## Step 2b — Full Write (Empty File Fallback) + + The file exists but its body is empty. Write the full content in one shot: + `write path={note_path} name= description= content=` + + - `name` must equal the filename stem of the target path (the part between the last `/` and `.md`), copied verbatim. + - `description` must be a thorough summary of the body — specific enough that the description alone conveys all key information. + + ## Step 3 — Summary + + State in one sentence what you did (what content was updated). This is your final text output. + + ## Boundaries + + - Only operate on one target path: `{note_path}`. Do not touch other notes. + - `write` unconditionally overwrites body and frontmatter — use with caution. +user_message_update_zh: | + 日期：{date} + Vault 目录：{vault_dir} + 资源文件：{file_path} + 目标笔记路径：{note_path} + + # 资源文件内容（已更新） + + {file_content} + + # 你的任务 + + 资源文件已更新。重新解读并更新目标路径的已有笔记。 + + ## 步骤 1 — 读取现有内容 + + 调用 `read path={note_path}` 查看当前笔记内容。 + - 如果正文为空（只有 frontmatter 无实际内容）→ 按新建处理，跳到 **步骤 2b**。 + - 如果有正文内容 → 转到 **步骤 2a** 进行更新。 + + ## 步骤 2a — 合并更新 + + 笔记已有来自资源文件旧版本的内容。你的任务是更新它以反映当前版本。 + + 更新规则： + - **已删除内容**：删除资源文件中不再存在的部分。 + - **新增内容**：为新增信息添加章节。 + - **修改内容**：重写受影响的部分以匹配当前文件。 + - **未变内容**：保持原样。 + + 执行： + 1. 对需要更新的每个部分使用 `edit path={note_path} old=<原文片段> new=<替换片段>`。可以多次调用 `edit`。 + 2. 正文变更后，刷新 frontmatter 的 description：`frontmatter_update path={note_path} metadata={{"description": "<更新后的总结>"}}`。 + 3. 如果 `edit` 多次失败（如找不到原文），退回 `write path={note_path} name= description= content=<完整正文>` 全量重写。 + + ## 步骤 2b — 全量写入（空文件 fallback） + + 文件存在但正文为空。一次性写入完整内容： + `write path={note_path} name= description= content=<正文>` + + - `name` 必须等于目标路径的文件名 stem（最后一个 `/` 与 `.md` 之间的部分），逐字照抄。 + - `description` 必须是正文的详尽总结——具体到仅凭 description 就能传达全部核心信息。 + + ## 步骤 3 — 总结 + + 用一句话说明你做了什么（更新了哪些内容）。这是你最后一次文本输出。 + + ## 边界 + + - 只针对一个目标路径：`{note_path}`。不要碰其他笔记。 + - `write` 会无条件覆盖正文和 frontmatter，请谨慎使用。 diff --git a/reme4/steps/evolve/dream.py b/reme4/steps/evolve/dream.py index 52008848..b302234a 100644 --- a/reme4/steps/evolve/dream.py +++ b/reme4/steps/evolve/dream.py @@ -47,18 +47,12 @@ Invocation form (CLI / MCP): reme dream path=resource/2026-05-28/spec.pdf hint="focus on auth" """ -import datetime -import zoneinfo from pathlib import Path from typing import Literal -from agentscope.agent import Agent -from agentscope.message import Msg, TextBlock -from agentscope.permission import PermissionContext, PermissionMode -from agentscope.state import AgentState -from agentscope.tool import Toolkit from pydantic import BaseModel, Field +from ._evolve import now from ..base_step import BaseStep from ...components import R @@ -245,24 +239,6 @@ class DreamStep(BaseStep): reme dream path=daily/2026-05-28/auth-refactor/auth-refactor.md """ - def __init__( - self, - toolkit: Toolkit | None = None, - timezone: str | None = None, - **kwargs, - ): - super().__init__(**kwargs) - self.toolkit = toolkit - self.timezone = timezone - - def _now(self) -> datetime.datetime: - if self.timezone: - try: - return datetime.datetime.now(zoneinfo.ZoneInfo(self.timezone)) - except Exception as e: - self.logger.error(f"Invalid timezone: {self.timezone}, error={e}") - return datetime.datetime.now() - def _vault_dir(self) -> Path: vr = getattr(self.file_store, "vault_path", None) return Path(vr).resolve() if vr else Path.cwd().resolve() @@ -273,25 +249,6 @@ class DreamStep(BaseStep): except Exception: return False - def _build_extract_toolkit(self) -> Toolkit: - """Read-only toolkit for the extract agent. Sub-units come back via - :class:`ExtractedUnits` structured output, not via a tool call.""" - toolkit = Toolkit() - for job_name in _EXTRACT_TOOLS: - self.add_as_tool(toolkit, job_name) - return toolkit - - def _build_integrate_toolkit(self) -> Toolkit: - """Full read + canonical write/edit/frontmatter_update toolkit for - the integrate agent. All tools are registered via :meth:`add_as_tool` - — same as every other step in this codebase. Outcome tracking is - driven by the agent's :class:`IntegrateOutcome` structured emission, - not by per-tool callbacks.""" - toolkit = self.toolkit or Toolkit() - for job_name in _INTEGRATE_TOOLS: - self.add_as_tool(toolkit, job_name) - return toolkit - async def _extract(self, material_blob: str, hint: str, vault_dir: Path) -> tuple[list[dict], str]: """Phase 1: one ReAct invocation — read material + emit ExtractedUnits. @@ -300,37 +257,27 @@ class DreamStep(BaseStep): and ``llm_summary`` is whatever free-form text the agent produced alongside its structured emission. """ - toolkit = self._build_extract_toolkit() - agent = Agent( - name="reme_dreamer_extract", - model=self.as_llm, + tools = [self.get_job(name) for name in _EXTRACT_TOOLS] + tz = self.app_context.app_config.timezone if self.app_context is not None else None + user_message = self.prompt_format( + "extract_user_message", + today=now(tz).strftime("%Y-%m-%d"), + hint=hint or "(none)", + material_blob=material_blob, + ) + _, result = await self.agent_wrapper.reply( + user_message, system_prompt=self.prompt_format( "extract_system_prompt", vault_dir=str(vault_dir), buckets=", ".join(BUCKETS), ), - toolkit=toolkit, - state=AgentState( - permission_context=PermissionContext( - mode=PermissionMode.BYPASS, - ), - ), - ) - user_message = self.prompt_format( - "extract_user_message", - today=self._now().strftime("%Y-%m-%d"), - hint=hint or "(none)", - material_blob=material_blob, - ) - msg = await agent.reply( - Msg(name="reme", role="user", content=[TextBlock(text=user_message)]), + tools=tools, + output_schema=ExtractedUnits, ) - structured_resp = await self.as_llm.generate_structured_output( - agent.state.context, - structured_model=ExtractedUnits, - ) - meta = structured_resp.content if isinstance(structured_resp.content, dict) else {} + msg = result["message"] + meta = result["structured_output"] if isinstance(result["structured_output"], dict) else {} cleaned: list[dict] = [] for raw in meta.get("units") or []: if not isinstance(raw, dict): @@ -341,8 +288,6 @@ class DreamStep(BaseStep): if not name or not summary: continue if bucket not in BUCKETS: - # Defensive: structured_model should already reject this, - # but if it slips through we route to wiki (the catch-all). self.logger.warning( f"[{self.name}] unit {name!r} emitted bucket {bucket!r} " f"not in {list(BUCKETS)}; routing to 'wiki'", @@ -358,24 +303,8 @@ class DreamStep(BaseStep): single source of truth for what got written (action + target_path).""" bucket = unit.get("bucket") or "wiki" - toolkit = self._build_integrate_toolkit() digest_dir = getattr(self.app_context.app_config, "digest_dir", "") - agent = Agent( - name=f"reme_dreamer_integrate_{unit.get('name', 'unit')}", - model=self.as_llm, - system_prompt=self.prompt_format( - f"integrate_system_prompt_{bucket}", - vault_dir=str(vault_dir), - digest_dir=digest_dir, - bucket=bucket, - ), - toolkit=toolkit, - state=AgentState( - permission_context=PermissionContext( - mode=PermissionMode.BYPASS, - ), - ), - ) + tools = [self.get_job(name) for name in _INTEGRATE_TOOLS] user_message = self.prompt_format( "integrate_user_message", hint=hint or "(none)", @@ -384,14 +313,18 @@ class DreamStep(BaseStep): unit_summary=unit.get("summary", ""), material_blob=material_blob, ) - await agent.reply( - Msg(name="reme", role="user", content=[TextBlock(text=user_message)]), + _, result = await self.agent_wrapper.reply( + user_message, + system_prompt=self.prompt_format( + f"integrate_system_prompt_{bucket}", + vault_dir=str(vault_dir), + digest_dir=digest_dir, + bucket=bucket, + ), + tools=tools, + output_schema=IntegrateOutcome, ) - structured_resp = await self.as_llm.generate_structured_output( - agent.state.context, - structured_model=IntegrateOutcome, - ) - return IntegrateOutcome.model_validate(structured_resp.content) + return IntegrateOutcome.model_validate(result["structured_output"]) async def dream_one(self, path: str, hint: str = "") -> DreamResult: """Run the full extract + integrate pipeline on one vault-relative diff --git a/reme4/steps/file_io/_daily_index.py b/reme4/steps/file_io/_daily_index.py index 1f7e3e45..f31677ee 100644 --- a/reme4/steps/file_io/_daily_index.py +++ b/reme4/steps/file_io/_daily_index.py @@ -53,8 +53,11 @@ def scan_notes(vault_dir: Path, date: str, daily_dir: str) -> list[dict]: if not date_dir.is_dir(): return [] out: list[dict] = [] - for md_path in sorted(p for p in date_dir.iterdir() if p.is_file() and p.suffix == ".md"): - session_id = md_path.stem + prefix = "session_agent_" + for md_path in sorted( + p for p in date_dir.iterdir() if p.is_file() and p.suffix == ".md" and p.stem.startswith(prefix) + ): + session_id = md_path.stem[len(prefix) :] try: post = frontmatter.loads(md_path.read_text(encoding="utf-8")) except Exception: @@ -62,7 +65,7 @@ def scan_notes(vault_dir: Path, date: str, daily_dir: str) -> list[dict]: out.append( { "session_id": session_id, - "path": f"{daily_dir}/{date}/{session_id}.md", + "path": f"{daily_dir}/{date}/{md_path.name}", "metadata": dict(post.metadata or {}), }, ) diff --git a/reme4/steps/file_io/daily_create.py b/reme4/steps/file_io/daily_create.py index 7f1ba1f8..cac75e8b 100644 --- a/reme4/steps/file_io/daily_create.py +++ b/reme4/steps/file_io/daily_create.py @@ -86,7 +86,7 @@ class DailyCreateStep(BaseStep): if err: self._fail(err) return None - path_rel = f"{daily_dir}/{day}/{session_id}.md" + path_rel = f"{daily_dir}/{day}/session_agent_{session_id}.md" name = session_id else: path_rel = f"{daily_dir}/{day}.md" diff --git a/reme4/steps/index/_watch_rules.py b/reme4/steps/index/_watch_rules.py new file mode 100644 index 00000000..631b4ada --- /dev/null +++ b/reme4/steps/index/_watch_rules.py @@ -0,0 +1,68 @@ +"""Shared watch-rule logic for scan_changes and watch_changes steps.""" + +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ...schema import ApplicationConfig + + +@dataclass +class WatchRule: + """A single directory-monitoring rule.""" + + path: Path + suffixes: list[str] = field(default_factory=list) + + +def build_watch_rules( + app_config: "ApplicationConfig", + vault_path: Path, + *, + watch_dirs: list[str], + watch_suffixes: list[str], +) -> list[WatchRule]: + """Build watch rules from application config fields and suffix whitelist.""" + rules: list[WatchRule] = [] + for dir_field in watch_dirs: + dir_name = getattr(app_config, dir_field, dir_field) + rules.append(WatchRule(path=vault_path / dir_name, suffixes=list(watch_suffixes))) + return rules + + +def collect_existing(rules: list[WatchRule], recursive: bool) -> dict[str, float]: + """Walk rule paths and return {abs_path: st_mtime} for matching files.""" + existing: dict[str, float] = {} + for rule in rules: + if not rule.path.exists(): + continue + candidates = rule.path.rglob("*") if recursive else rule.path.iterdir() + for p in candidates: + if not p.is_file(): + continue + if not _match_rule(p, rule): + continue + abs_p = p.absolute() + existing[str(abs_p)] = abs_p.stat().st_mtime + return existing + + +def match_file(file_path: str, rules: list[WatchRule]) -> bool: + """Return True if a file path matches any of the watch rules.""" + p = Path(file_path) + for rule in rules: + try: + p.relative_to(rule.path) + except ValueError: + continue + if _match_rule(p, rule): + return True + return False + + +def _match_rule(p: Path, rule: WatchRule) -> bool: + """Check if a single path matches a rule's suffix constraint.""" + if rule.suffixes and not any(p.name.endswith("." + s.strip(".")) for s in rule.suffixes): + return False + return True diff --git a/reme4/steps/index/foreach_dispatch.py b/reme4/steps/index/foreach_dispatch.py new file mode 100644 index 00000000..c2f6aa2f --- /dev/null +++ b/reme4/steps/index/foreach_dispatch.py @@ -0,0 +1,27 @@ +"""Foreach dispatch: iterate changes and call a configured job per file.""" + +from ..base_step import BaseStep +from ...components import R + + +@R.register("foreach_dispatch_step") +class ForeachDispatchStep(BaseStep): + """For each change item, call ``dispatch_job`` with the vault-relative path.""" + + async def execute(self): + assert self.context is not None + changes: list[dict] = self.context.get("changes") or [] + dispatch_job: str = self.context.get("dispatch_job", "") + if not dispatch_job: + self.logger.warning(f"[{self.name}] no dispatch_job configured, skip") + self.context.response.success = True + return self.context.response + for item in changes: + rel_path = self.to_vault_relative(item["path"]) + try: + await self.run_job(dispatch_job, file_path=rel_path, change=item["change"]) + except Exception: + self.logger.exception(f"[{self.name}] dispatch {dispatch_job} failed: {rel_path}") + self.context.response.success = True + self.context.response.metadata["dispatched"] = len(changes) + return self.context.response diff --git a/reme4/steps/index/log_changes.py b/reme4/steps/index/log_changes.py new file mode 100644 index 00000000..b091dc1e --- /dev/null +++ b/reme4/steps/index/log_changes.py @@ -0,0 +1,18 @@ +"""Log changes step: mock dispatch target that logs detected changes.""" + +from ..base_step import BaseStep +from ...components import R + + +@R.register("log_changes_step") +class LogChangesStep(BaseStep): + """Log each change item. Placeholder for future digest-watch logic.""" + + async def execute(self): + assert self.context is not None + changes: list[dict] = self.context.get("changes") or [] + for item in changes: + self.logger.info(f"[{self.name}] {item['change']}: {item['path']}") + self.context.response.success = True + self.context.response.metadata["count"] = len(changes) + return self.context.response diff --git a/reme4/steps/index/scan_changes.py b/reme4/steps/index/scan_changes.py index 2a8c8c6e..c173a07a 100644 --- a/reme4/steps/index/scan_changes.py +++ b/reme4/steps/index/scan_changes.py @@ -5,11 +5,11 @@ write target so the index loop and the dream loop never contend on the same component: * :class:`ScanStoreChangesStep` (``scan_store_changes_step``) — diffs - against ``file_store``; used by ``update_store_index_loop`` (sole - writer of ``file_store``). + against ``file_store``; used by ``index_update_loop`` (sole writer of + ``file_store``). * :class:`ScanCatalogChangesStep` (``scan_catalog_changes_step``) — diffs - against ``file_catalog``; used by ``auto_dream_loop`` (sole writer of - ``file_catalog``). + against ``file_catalog``; used by ``resource_watch_loop`` and + ``digest_watch_loop`` (sole writers of ``file_catalog``). Both share the same diff vocabulary (``added`` / ``modified`` / ``deleted``) and write into ``context["changes"]`` in the same shape, so downstream @@ -19,50 +19,20 @@ steps don't care which variant produced the batch. from pathlib import Path from typing import Iterable -from ..base_step import BaseStep, Ref +from ._watch_rules import WatchRule, build_watch_rules, collect_existing +from ..base_step import BaseStep from ...components import R -from ...components.file_catalog import BaseFileCatalog -from ...enumeration import ComponentEnum from ...schema import FileNode -def _collect_existing( - raw: list[str] | str, - suffixes: list[str], - vault_path: Path, - recursive: bool, -) -> dict[str, float]: - """Walk watch_paths under ``vault_path`` and return ``{abs_path: st_mtime}``.""" - paths = [raw] if isinstance(raw, str) else raw - watch_paths = [vault_path / x for x in paths if (vault_path / x).exists()] - - existing: dict[str, float] = {} - for path in watch_paths: - candidates = [path] if path.is_file() else (path.rglob("*") if recursive else path.iterdir()) - for p in candidates: - if not p.is_file(): - continue - if suffixes and not any(str(p).endswith("." + s.strip(".")) for s in suffixes): - continue - abs_p = p.absolute() - existing[str(abs_p)] = abs_p.stat().st_mtime - return existing - - -def _diff( - existing: dict[str, float], - nodes: Iterable[FileNode], - vault_path: Path, -) -> tuple[list[dict], dict[str, int]]: +def _diff(existing: dict[str, float], nodes: Iterable[FileNode], vault_path: Path) -> tuple[list[dict], dict[str, int]]: """Compute added/modified/deleted vs ``nodes`` and return (changes, counts).""" indexed: dict[str, float] = { str(Path(n.path) if Path(n.path).is_absolute() else vault_path / n.path): n.st_mtime for n in nodes } - to_delete = list(indexed.keys() - existing.keys()) to_add = list(existing.keys() - indexed.keys()) to_modify = [p for p in existing.keys() & indexed.keys() if existing[p] != indexed[p]] - changes: list[dict] = ( [{"change": "added", "path": p} for p in to_add] + [{"change": "modified", "path": p} for p in to_modify] @@ -79,31 +49,32 @@ class _ScanChangesBase(BaseStep): super().__init__(**kwargs) self.recursive: bool = recursive + def _get_watch_rules(self) -> list[WatchRule]: + """Build watch rules from context-level watch_dirs/watch_suffixes.""" + assert self.context is not None + app_config = self.app_context.app_config if self.app_context else None + if app_config is None: + return [] + watch_dirs: list[str] = self.context.get("watch_dirs", []) + watch_suffixes: list[str] = self.context.get("watch_suffixes", []) + if not watch_dirs: + return [] + return build_watch_rules(app_config, self.vault_path, watch_dirs=watch_dirs, watch_suffixes=watch_suffixes) + async def _load_indexed_nodes(self) -> Iterable[FileNode]: raise NotImplementedError async def execute(self): assert self.context is not None - vault_path = self.vault_path - raw: list[str] = self.context.get("watch_paths", []) or [] - suffixes: list[str] = self.context.get("suffix_filters", ["md"]) or ["md"] - - existing = _collect_existing( - raw=raw, - suffixes=suffixes, - vault_path=vault_path, - recursive=self.recursive, - ) - + rules = self._get_watch_rules() + existing = collect_existing(rules, recursive=self.recursive) nodes = await self._load_indexed_nodes() - changes, counts = _diff(existing, nodes, vault_path) - + changes, counts = _diff(existing, nodes, self.vault_path) self.context["changes"] = changes if changes: self.logger.info(f"[{self.name}] scan: {counts}") else: self.logger.info(f"[{self.name}] store is up to date") - self.context.response.metadata["counts"] = counts return self.context.response @@ -120,9 +91,7 @@ class ScanStoreChangesStep(_ScanChangesBase): @R.register("scan_catalog_changes_step") class ScanCatalogChangesStep(_ScanChangesBase): - """Diff vault against ``file_catalog``; used by the dream loop.""" - - file_catalog: BaseFileCatalog = Ref(BaseFileCatalog, ComponentEnum.FILE_CATALOG) + """Diff vault against ``file_catalog``; used by resource/digest loops.""" async def _load_indexed_nodes(self) -> Iterable[FileNode]: return await self.file_catalog.get_nodes() diff --git a/reme4/steps/index/update_catalog.py b/reme4/steps/index/update_catalog.py index 56936919..627c6886 100644 --- a/reme4/steps/index/update_catalog.py +++ b/reme4/steps/index/update_catalog.py @@ -4,10 +4,8 @@ from pathlib import Path from watchfiles import Change -from ..base_step import BaseStep, Ref +from ..base_step import BaseStep from ...components import R -from ...components.file_catalog import BaseFileCatalog -from ...enumeration import ComponentEnum from ...schema import FileNode @@ -15,13 +13,14 @@ from ...schema import FileNode class UpdateCatalogStep(BaseStep): """Classify raw watcher changes and update the file_catalog.""" - file_catalog: BaseFileCatalog = Ref(BaseFileCatalog, ComponentEnum.FILE_CATALOG) + def __init__(self, persist: bool = False, **kwargs): + super().__init__(**kwargs) + self._persist: bool = persist async def execute(self): assert self.context is not None - # Each item: {"change": Change | "added"|"modified"|"deleted", "path": absolute path} changes: list[dict] = self.context.get("changes") or [] - persist: bool = bool(self.context.get("persist", False)) + persist: bool = self._persist or bool(self.context.get("persist", False)) buckets: dict[Change, list[str]] = {Change.added: [], Change.modified: [], Change.deleted: []} for item in changes: diff --git a/reme4/steps/index/update_index.py b/reme4/steps/index/update_index.py index 7bcb9446..b8d1a258 100644 --- a/reme4/steps/index/update_index.py +++ b/reme4/steps/index/update_index.py @@ -6,6 +6,8 @@ from watchfiles import Change from ..base_step import BaseStep from ...components import R +from ...components.file_chunker import BaseFileChunker +from ...enumeration import ComponentEnum from ...schema import FileChunk, FileNode @@ -17,6 +19,35 @@ class UpdateIndexStep(BaseStep): super().__init__(**kwargs) self.persist: bool = persist + async def chunk_file(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]: + """Chunk ``path`` using the file chunker whose ``supported_extensions`` claims its suffix. + + First registered match wins (config insertion order). Falls back to the + ``default`` chunker when no chunker claims the suffix. + """ + if self.app_context is None: + raise RuntimeError("app_context is not set when resolving file chunker") + chunker_dict: dict[str, BaseFileChunker] = self.app_context.components[ComponentEnum.FILE_CHUNKER] + + suffix = Path(path).suffix.lstrip(".").lower() + + chunker: BaseFileChunker | None = None + if suffix: + for candidate in chunker_dict.values(): + if suffix in {ext.lower().lstrip(".") for ext in candidate.supported_extensions}: + chunker = candidate + break + + if chunker is None: + chunker = chunker_dict.get("default") + + if chunker is None: + raise RuntimeError( + f"No file chunker supports {path} (suffix={suffix!r}) and no 'default' chunker is configured", + ) + + return await chunker.chunk(path) + async def execute(self): assert self.context is not None # Each item: {"change": Change | "added"|"modified"|"deleted", "path": absolute path} @@ -46,7 +77,7 @@ class UpdateIndexStep(BaseStep): continue self.logger.info(f"{action} file: {path}") try: - parsed.append(await self.parse_file(abs_path)) + parsed.append(await self.chunk_file(abs_path)) ok_paths.append(path) except Exception as e: self.logger.exception(f"Failed to parse {path}") diff --git a/reme4/steps/index/watch_changes.py b/reme4/steps/index/watch_changes.py index 5f88401c..1b50808a 100644 --- a/reme4/steps/index/watch_changes.py +++ b/reme4/steps/index/watch_changes.py @@ -6,7 +6,7 @@ Two relevant awatch parameters are exposed verbatim: has gone this long without new changes (and at least one change is pending). Raise to ``5 minutes``-ish for ``auto_dream_loop`` so half-written sync output isn't dreamed mid-write; keep at default - for ``update_store_index_loop`` where every fs change should hit + for ``index_update_loop`` where every fs change should hit the index promptly. * ``debounce`` (default ``2000ms``) — per-batch ceiling, regardless @@ -14,8 +14,8 @@ Two relevant awatch parameters are exposed verbatim: ``step`` is the operative limit; otherwise the watcher pre-empts long-quiet-window setups under bursty writes. -Both are global to the watcher (not per-path). The two reme watchers -have disjoint ``watch_paths`` (digest vs daily/resource), so global +Both are global to the watcher (not per-path). The reme watchers +have disjoint ``watch_dirs`` (configured per job), so global quiet windows are good enough — no per-path bookkeeping needed. awatch internally deduplicates same-path same-change tuples within @@ -27,6 +27,7 @@ import asyncio from watchfiles import Change, awatch +from ._watch_rules import WatchRule, build_watch_rules, match_file from ..base_step import BaseStep from ...components import R, BaseComponent from ...enumeration import ComponentEnum @@ -34,7 +35,7 @@ from ...enumeration import ComponentEnum @R.register("watch_changes_step") class WatchChangesStep(BaseStep): - """Watch files and forward each yielded batch to a downstream step.""" + """Watch files and forward each yielded batch to downstream steps.""" def __init__( self, @@ -44,6 +45,7 @@ class WatchChangesStep(BaseStep): step: int = 50, poll_delay_ms: int = 2000, dispatch_step: str = "", + dispatch_steps: list[str] | None = None, **kwargs, ): super().__init__(**kwargs) @@ -52,11 +54,23 @@ class WatchChangesStep(BaseStep): self.debounce: int = debounce self.step: int = step self.poll_delay_ms: int = poll_delay_ms - self.dispatch_step: str = dispatch_step + self.dispatch_steps: list[str] = dispatch_steps or ([dispatch_step] if dispatch_step else []) + self._rules: list[WatchRule] = [] + + def _get_watch_rules(self) -> list[WatchRule]: + """Build watch rules from context-level watch_dirs/watch_suffixes.""" + assert self.context is not None + app_config = self.app_context.app_config if self.app_context else None + if app_config is None: + return [] + watch_dirs: list[str] = self.context.get("watch_dirs", []) + watch_suffixes: list[str] = self.context.get("watch_suffixes", []) + if not watch_dirs: + return [] + return build_watch_rules(app_config, self.vault_path, watch_dirs=watch_dirs, watch_suffixes=watch_suffixes) def _filter(self, _change: Change, path: str) -> bool: - suffixes = (self.context.get("suffix_filters") if self.context else None) or ["md"] - return not suffixes or any(path.endswith("." + s.strip(".")) for s in suffixes) + return match_file(path, self._rules) async def execute(self): if self.context is None: @@ -65,21 +79,22 @@ class WatchChangesStep(BaseStep): raise RuntimeError("watch_changes_step requires 'stop_event' on context") stop_event: asyncio.Event = self.context.stop_event - raw = self.context.get("watch_paths", []) - paths = [raw] if isinstance(raw, str) else raw - valid_paths = [self.vault_path / x for x in paths if (self.vault_path / x).exists()] + self._rules = self._get_watch_rules() + if not self._rules: + raise RuntimeError("No watch rules configured (watch_dirs empty or app_config missing?)") + + valid_paths = list(dict.fromkeys(r.path for r in self._rules if r.path.exists())) if not valid_paths: - raise RuntimeError(f"No valid watch paths under {self.vault_path}: {paths}") + raise RuntimeError(f"No valid watch paths exist: {[str(r.path) for r in self._rules]}") - dispatch_step_cls: type[BaseComponent] | None = None - if self.dispatch_step: - dispatch_step_cls = R.get(ComponentEnum.STEP, self.dispatch_step) - if dispatch_step_cls is None: - raise RuntimeError(f"Unregistered step '{self.dispatch_step}'") + dispatch_classes: list[type[BaseComponent]] = [] + for name in self.dispatch_steps: + cls = R.get(ComponentEnum.STEP, name) + if cls is None: + raise RuntimeError(f"Unregistered step '{name}'") + dispatch_classes.append(cls) - self.logger.info( - f"Watching: {[str(p) for p in valid_paths]} step={self.step}ms debounce={self.debounce}ms", - ) + self.logger.info(f"Watching: {[str(p) for p in valid_paths]} step={self.step}ms debounce={self.debounce}ms") async for raw_changes in awatch( *valid_paths, @@ -100,8 +115,9 @@ class WatchChangesStep(BaseStep): ] if changes: self.logger.info(f"Detected {len(changes)} change(s)") - if dispatch_step_cls is not None: - step = dispatch_step_cls(app_context=self.app_context) - await step(changes=changes) + extra = {k: v for k, v in self.context.data.items() if k not in ("stop_event",)} + for cls in dispatch_classes: + s = cls(app_context=self.app_context) + await s(changes=changes, **extra) return self.context.response diff --git a/reme4/utils/__init__.py b/reme4/utils/__init__.py index 2bbff23f..47fa5ab1 100644 --- a/reme4/utils/__init__.py +++ b/reme4/utils/__init__.py @@ -14,6 +14,7 @@ from .logo_utils import print_logo from .service_utils import find_reme, locate_reme, precheck_start, cli_find_reme from .similarity_utils import cosine_similarity, batch_cosine_similarity from .token_utils import estimate_token_count +from .agent_state_io import AsStateHandler __all__ = [ "hash_text", @@ -33,4 +34,5 @@ __all__ = [ "cosine_similarity", "batch_cosine_similarity", "estimate_token_count", + "AsStateHandler", ] diff --git a/reme4/utils/agent_state_io.py b/reme4/utils/agent_state_io.py new file mode 100644 index 00000000..9720723e --- /dev/null +++ b/reme4/utils/agent_state_io.py @@ -0,0 +1,54 @@ +"""AgentState JSONL dump / load. + +Format: + Line 1 — header Msg: AgentState.summary as content, state scalars in metadata. + Lines 2+ — AgentState.context, one Msg per line. +""" + +from pathlib import Path + +import aiofiles +from agentscope.message import Msg, UserMsg +from agentscope.state import AgentState + +_META_KEYS = ("session_id", "reply_id", "cur_iter") + + +class AsStateHandler: + """Serialize / deserialize AgentState to a JSONL file.""" + + def __init__(self, path: str | Path): + self.path = Path(path) + + async def dump(self, state: AgentState) -> Path: + """Write *state* to ``self.path`` in JSONL format.""" + header = UserMsg( + name="__state__", + content=state.summary or "", + metadata={k: getattr(state, k) for k in _META_KEYS}, + ) + async with aiofiles.open(self.path, "w", encoding="utf-8") as f: + await f.write(header.model_dump_json() + "\n") + for msg in state.context: + await f.write(msg.model_dump_json() + "\n") + return self.path + + async def load(self) -> AgentState: + """Read an AgentState back from ``self.path``.""" + async with aiofiles.open(self.path, encoding="utf-8") as f: + lines = (await f.read()).splitlines() + if not lines: + return AgentState() + + header = Msg.model_validate_json(lines[0]) + summary: str | list = ( + list(header.content) + if any(getattr(b, "type", None) == "data" for b in header.content) + else header.get_text_content() or "" + ) + + return AgentState( + **{k: header.metadata.get(k, d) for k, d in [("session_id", ""), ("reply_id", ""), ("cur_iter", 0)]}, + summary=summary, + context=[Msg.model_validate_json(line) for line in lines[1:] if line.strip()], + ) diff --git a/reme4/utils/transcript.py b/reme4/utils/transcript.py deleted file mode 100644 index cbc96ebf..00000000 --- a/reme4/utils/transcript.py +++ /dev/null @@ -1,195 +0,0 @@ -"""Parse Claude Code transcript JSONL into a plain message slice. - -Both routes (``reme-driver`` external, ``reme-service`` internal) accept -``transcript_path`` as the canonical ``sync`` input — Claude Code hooks -hand us the path, not the message list. Centralising the parse here -keeps the two routes producing identical slices for the same transcript. - -The JSONL format is one record per line. The records we care about: - - {"type": "user", "message": {"role": "user", "content": str | list[block]}, ...} - {"type": "assistant", "message": {"role": "assistant", "content": list[block]}, ...} - -Everything else (``ai-title`` / ``mode`` / ``permission-mode`` / -``file-history-snapshot`` / ``attachment`` / ``last-prompt`` / -``queue-operation`` / ``system``) is metadata or harness chatter and is -ignored. - -Content blocks we recognise (shape from the Anthropic message format): - -* ``{"type": "text", "text": str}`` — appended verbatim -* ``{"type": "tool_use", "name": str, "input": ...}`` — rendered as ``[tool ()]`` -* ``{"type": "tool_result", "content": ...}`` — rendered as ``[tool_result ]`` -* ``{"type": "thinking", "thinking": str}`` — dropped (private reasoning) - -User content frequently contains Claude-Code-injected boilerplate that -isn't part of the real conversation: - -* ``...`` — `bash` command warnings prepended to first user turn -* ``...`` — output of slash commands -* ``...`` — slash command label -* ``...`` — slash command description -* ``...`` — periodic harness reminders - -These are filtered out (whole-message drop if the text is only injected -markers, partial strip otherwise) so the synchronizer sees the actual -user/assistant dialogue. -""" - -from __future__ import annotations - -import json -import re -from pathlib import Path -from typing import Any - -# Whole-message-drop when the user content is only one of these wrappers. -_INJECTED_TAGS = ( - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", -) - -# Heuristic: if the message starts with one of these AND is short / mostly markup, -# drop it. We keep the regex permissive — false negatives (a real message that -# looks like markup) are recoverable downstream; false positives (dropping real -# user text) are silent and worse. -_DROP_IF_STARTS_WITH = tuple(_INJECTED_TAGS) - - -def load_messages_from_transcript( - transcript_path: str | Path, - *, - include_thinking: bool = False, - tool_input_excerpt: int = 200, -) -> list[dict[str, str]]: - """Read a Claude Code transcript JSONL → list of role/content dicts. - - Returns ``[{role, name, content}, ...]`` in source order, where - ``role`` is ``"user"`` or ``"assistant"`` and ``name`` mirrors role - (so the dicts are directly consumable by AgentScope's ``Msg``, which - requires a ``name`` field). Empty list when the file is missing, - empty, or contains no user/assistant turns. - - Parameters - ---------- - transcript_path: - Absolute or relative path to the transcript JSONL file. - include_thinking: - When True, include ``thinking`` blocks (assistant private reasoning). - Default False — the synchronizer wants observable dialogue. - tool_input_excerpt: - Max chars of a ``tool_use`` input JSON to render inline. Default 200. - """ - path = Path(transcript_path) - if not path.is_file(): - return [] - - messages: list[dict[str, str]] = [] - for line in path.read_text(encoding="utf-8").splitlines(): - line = line.strip() - if not line: - continue - try: - record = json.loads(line) - except json.JSONDecodeError: - continue - - record_type = record.get("type") - if record_type not in ("user", "assistant"): - continue - - message = record.get("message") or {} - role = message.get("role") - if role not in ("user", "assistant"): - continue - - text = _render_content( - message.get("content", ""), - include_thinking=include_thinking, - tool_input_excerpt=tool_input_excerpt, - ) - if not text: - continue - if _is_injected_only(text): - continue - - messages.append({"role": role, "name": role, "content": text}) - - return messages - - -def _render_content( - content: Any, - *, - include_thinking: bool, - tool_input_excerpt: int, -) -> str: - if isinstance(content, str): - return content.strip() - - if not isinstance(content, list): - return "" - - parts: list[str] = [] - for block in content: - if not isinstance(block, dict): - continue - btype = block.get("type") - - if btype == "text": - t = (block.get("text") or "").strip() - if t: - parts.append(t) - - elif btype == "thinking": - if include_thinking: - t = (block.get("thinking") or "").strip() - if t: - parts.append(f"[thinking]\n{t}") - - elif btype == "tool_use": - name = block.get("name", "?") - try: - inp = json.dumps(block.get("input"), ensure_ascii=False)[:tool_input_excerpt] - except (TypeError, ValueError): - inp = str(block.get("input"))[:tool_input_excerpt] - parts.append(f"[tool {name}({inp})]") - - elif btype == "tool_result": - inner = block.get("content") - if isinstance(inner, list): - excerpt = _render_content( - inner, - include_thinking=False, - tool_input_excerpt=tool_input_excerpt, - ) - else: - excerpt = str(inner or "") - excerpt = excerpt.strip() - if len(excerpt) > tool_input_excerpt: - excerpt = excerpt[:tool_input_excerpt] + "..." - parts.append(f"[tool_result {excerpt}]") - - return "\n".join(p for p in parts if p).strip() - - -def _is_injected_only(text: str) -> bool: - """True if the text is composed entirely of Claude-Code-injected markers - (no genuine user/assistant dialogue around them). - """ - stripped = text.strip() - if not stripped.startswith(_DROP_IF_STARTS_WITH): - return False - # If it starts with an injected tag, peek whether anything substantive - # follows the closing tag. Cheap heuristic: strip all wrapped ... - # blocks and see what's left. - remaining = re.sub(r"<([a-z-]+)>.*?", "", stripped, flags=re.DOTALL) - return len(remaining.strip()) < 16 # arbitrary "essentially empty" threshold diff --git a/tests4/integration/test_agent_session.py b/tests4/integration/test_agent_session.py new file mode 100644 index 00000000..446e4926 --- /dev/null +++ b/tests4/integration/test_agent_session.py @@ -0,0 +1,170 @@ +"""Integration tests: session state persistence and forking in AsAgentWrapper. + +Requires LLM_API_KEY (and optionally LLM_BASE_URL / LLM_MODEL_NAME) in the +environment or a .env file at the repo root. Hits the real LLM API. +""" + +import asyncio +import os +import tempfile +from pathlib import Path + +from reme4 import Application +from reme4.config import resolve_app_config +from reme4.enumeration import ComponentEnum +from reme4.utils import load_env + +load_env() + + +class _temp_chdir: + def __init__(self, path): + self.path = path + self._old = None + + def __enter__(self): + self._old = os.getcwd() + os.chdir(self.path) + return self + + def __exit__(self, *exc): + os.chdir(self._old) + + +async def _make_app() -> Application: + cfg = resolve_app_config(log_to_console=False, log_to_file=False, enable_logo=False) + app = Application(**cfg) + await app.start() + return app + + +def _find_session_files(vault_root: Path, prefix: str = "session_reme_") -> list[Path]: + resource_dir = vault_root / "resource" + if not resource_dir.exists(): + return [] + return sorted(resource_dir.rglob(f"{prefix}*.jsonl")) + + +async def _run_session_persistence() -> None: + """Two consecutive replies with the same session_id should share context.""" + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + app = await _make_app() + try: + vault_root = Path(app.config.vault_dir).absolute() + wrapper = app.context.components[ComponentEnum.AGENT_WRAPPER]["default"] + + sid = "test-persist-session" + + # First call: establish session + _, msg_1 = await wrapper.reply( + "My favorite color is blue. Remember that.", + session_id=sid, + system_prompt="You are a helpful assistant. Keep answers short.", + ) + text_1 = (msg_1.get_text_content() or "").strip() + print(f"\n[session_persist] reply 1: {text_1!r}") + assert text_1, "Empty first reply" + + # Verify session file was created + files_after_1 = _find_session_files(vault_root) + print(f"[session_persist] session files after reply 1: {files_after_1}") + assert len(files_after_1) == 1, f"Expected 1 session file, got {len(files_after_1)}" + assert sid in files_after_1[0].name + + # Second call: same session_id — agent should have previous context + _, msg_2 = await wrapper.reply( + "What is my favorite color?", + session_id=sid, + system_prompt="You are a helpful assistant. Keep answers short.", + ) + text_2 = (msg_2.get_text_content() or "").strip() + print(f"[session_persist] reply 2: {text_2!r}") + assert "blue" in text_2.lower(), f"Agent should recall 'blue' from session context, got: {text_2!r}" + + print("✓ test_session_persistence passed") + finally: + await app.close() + + +async def _run_fork_session() -> None: + """fork_session=True should create a new session file with a new session_id.""" + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + app = await _make_app() + try: + vault_root = Path(app.config.vault_dir).absolute() + wrapper = app.context.components[ComponentEnum.AGENT_WRAPPER]["default"] + + sid = "test-fork-origin" + + # Establish original session + await wrapper.reply( + "The secret number is 42.", + session_id=sid, + system_prompt="You are a helpful assistant. Keep answers short.", + ) + files_before_fork = _find_session_files(vault_root) + assert len(files_before_fork) == 1 + + # Fork the session + forked_sid, msg_fork = await wrapper.reply( + "What is the secret number?", + session_id=sid, + fork_session=True, + system_prompt="You are a helpful assistant. Keep answers short.", + ) + text_fork = (msg_fork.get_text_content() or "").strip() + print(f"\n[fork_session] forked reply: {text_fork!r}") + assert "42" in text_fork, f"Forked session should recall '42', got: {text_fork!r}" + + # Verify: original file still exists + new forked file created + files_after_fork = _find_session_files(vault_root) + print(f"[fork_session] session files after fork: {[f.name for f in files_after_fork]}") + assert ( + len(files_after_fork) == 2 + ), f"Expected 2 session files (original + fork), got {len(files_after_fork)}" + + # Forked session_id should differ from the original + assert forked_sid != sid, f"Forked session_id should differ from original, got {forked_sid!r}" + + original_file = vault_root / "resource" / files_before_fork[0].relative_to(vault_root / "resource") + assert original_file.exists(), "Original session file should still exist after fork" + + print("✓ test_fork_session passed") + finally: + await app.close() + + +async def _run_no_session_id() -> None: + """When session_id is empty, no session file should be created.""" + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + app = await _make_app() + try: + vault_root = Path(app.config.vault_dir).absolute() + wrapper = app.context.components[ComponentEnum.AGENT_WRAPPER]["default"] + + _, msg = await wrapper.reply( + "Say hello.", + system_prompt="You are a helpful assistant. Keep answers short.", + ) + text = (msg.get_text_content() or "").strip() + print(f"\n[no_session] reply: {text!r}") + assert text, "Empty reply" + + files = _find_session_files(vault_root) + assert len(files) == 0, f"No session files should be created without session_id, found {files}" + + print("✓ test_no_session_id passed") + finally: + await app.close() + + +async def _run_all() -> None: + await _run_no_session_id() + await _run_session_persistence() + await _run_fork_session() + + +if __name__ == "__main__": + print("=== Agent session state integration tests ===") + asyncio.run(_run_all()) + print("\nAll integration tests passed!") diff --git a/tests4/integration/test_auto_memory.py b/tests4/integration/test_auto_memory.py index 13b7d52d..653642ff 100644 --- a/tests4/integration/test_auto_memory.py +++ b/tests4/integration/test_auto_memory.py @@ -28,7 +28,7 @@ from reme4.utils import load_env load_env() -DUMP_DIR = Path(__file__).resolve().parent +DUMP_DIR = Path(__file__).resolve().parent / "agent_logs" SEED_STEM = "auth-middleware-rewrite" SEED_BODY = """--- @@ -200,6 +200,7 @@ class _AgentMemoryRecorder: async def dump(self) -> list[Path]: """Dump all agent context histories.""" + self.dump_dir.mkdir(parents=True, exist_ok=True) for stale in self.dump_dir.glob(f"{self.prefix}_*.jsonl"): stale.unlink() diff --git a/tests4/integration/test_auto_resource.py b/tests4/integration/test_auto_resource.py new file mode 100644 index 00000000..44402080 --- /dev/null +++ b/tests4/integration/test_auto_resource.py @@ -0,0 +1,361 @@ +"""Integration test for the auto_resource job. + +Drives the ``auto_resource`` step against a real LLM. Three scenarios: + +1. **CREATE (added)**: places a resource file in ``resource/{date}/``, + calls ``auto_resource`` with change="added". Expects a new note + ``daily/{date}/resource_{hash}.md`` with key facts from the file. + +2. **UPDATE (modified)**: seeds an existing resource note, updates the + resource file, calls ``auto_resource`` with change="modified". + Expects the note to reflect the updated content. + +3. **DELETE (deleted)**: seeds a resource note, calls ``auto_resource`` + with change="deleted". Expects the note file to be removed. + +Requires LLM_API_KEY (and optionally LLM_BASE_URL / LLM_MODEL_NAME) in the +environment or a .env file at the repo root. Hits the real LLM API. +""" + +import asyncio +import json +import os +import tempfile +from datetime import date as _date +from pathlib import Path + +from agentscope.agent import Agent + +from reme4 import Application +from reme4.config import resolve_app_config +from reme4.steps.evolve.auto_resource import _compute_session_id +from reme4.utils import load_env + +load_env() + +DUMP_DIR = Path(__file__).resolve().parent / "agent_logs" + +RESOURCE_FILENAME = "project-roadmap.md" +RESOURCE_CONTENT_V1 = """\ +# Project Roadmap 2026 Q3 + +## Goals +- Launch v2.0 API by July 15 +- Migrate 80% of users to new auth system by August 1 +- Reduce p99 latency to < 200ms + +## Milestones +| Date | Milestone | Owner | +|------------|------------------------|---------| +| 2026-07-01 | API beta release | Alice | +| 2026-07-15 | API GA | Alice | +| 2026-08-01 | Auth migration done | Bob | +| 2026-08-15 | Performance target met | Charlie | + +## Risks +- Auth migration blocked on legacy client deprecation (ETA: June 30) +- Performance target requires Redis cluster upgrade (budget approved) +""" + +RESOURCE_CONTENT_V2 = """\ +# Project Roadmap 2026 Q3 (Revised) + +## Goals +- Launch v2.0 API by July 20 (delayed 5 days from original July 15) +- Migrate 80% of users to new auth system by August 1 +- Reduce p99 latency to < 150ms (tightened from 200ms) + +## Milestones +| Date | Milestone | Owner | +|------------|------------------------|---------| +| 2026-07-05 | API beta release | Alice | +| 2026-07-20 | API GA | Alice | +| 2026-08-01 | Auth migration done | Bob | +| 2026-08-15 | Performance target met | Charlie | +| 2026-08-20 | Post-launch review | Dave | + +## Risks +- Auth migration blocked on legacy client deprecation (resolved June 28) +- Performance target requires Redis cluster upgrade (completed July 1) +- New risk: third-party OAuth provider rate limiting during migration +""" + + +def _today() -> str: + return _date.today().isoformat() + + +class _temp_chdir: + def __init__(self, path): + self.path = path + self._old = None + + def __enter__(self): + self._old = os.getcwd() + os.chdir(self.path) + return self + + def __exit__(self, *exc): + os.chdir(self._old) + + +async def _make_app() -> Application: + cfg = resolve_app_config(log_to_console=False, log_to_file=False, enable_logo=False) + app = Application(**cfg) + await app.start() + return app + + +def _place_resource(vault_root: Path, today: str, filename: str, content: str) -> str: + """Write a resource file and return its vault-relative path.""" + resource_dir = vault_root / "resource" / today + resource_dir.mkdir(parents=True, exist_ok=True) + path = resource_dir / filename + path.write_text(content, encoding="utf-8") + return f"resource/{today}/{filename}" + + +def _seed_resource_note(vault_root: Path, today: str, session_id: str, body: str) -> Path: + """Pre-seed a resource note in daily/{date}/.""" + day_dir = vault_root / "daily" / today + day_dir.mkdir(parents=True, exist_ok=True) + path = day_dir / f"{session_id}.md" + path.write_text(body, encoding="utf-8") + return path + + +def _read_text(p: Path) -> str: + return p.read_text(encoding="utf-8") + + +class _AgentMemoryRecorder: + """Monkey-patches Agent.__init__ to capture every agent created inside + the ``with`` block, then dumps each agent's memory to a jsonl file. + """ + + def __init__(self, dump_dir: Path, prefix: str = "agent_memory"): + self.dump_dir = dump_dir + self.prefix = prefix + self.agents: list[Agent] = [] + self._orig_init = None + self.dumped_paths: list[Path] = [] + + def __enter__(self): + self._orig_init = Agent.__init__ + agents = self.agents + orig = self._orig_init + + def _capturing_init(agent_self, *args, **kwargs): + orig(agent_self, *args, **kwargs) + agents.append(agent_self) + + Agent.__init__ = _capturing_init + return self + + def __exit__(self, *exc): + Agent.__init__ = self._orig_init + + async def dump(self) -> list[Path]: + """Serialize captured agent transcripts to disk.""" + self.dump_dir.mkdir(parents=True, exist_ok=True) + for stale in self.dump_dir.glob(f"{self.prefix}_*.jsonl"): + stale.unlink() + + for idx, agent in enumerate(self.agents, 1): + messages = agent.state.context + name = getattr(agent, "name", "agent") or "agent" + out_path = self.dump_dir / f"{self.prefix}_{idx:02d}_{name}.jsonl" + with out_path.open("w", encoding="utf-8") as f: + for msg in messages: + f.write(json.dumps(msg.model_dump(), ensure_ascii=False, default=str) + "\n") + self.dumped_paths.append(out_path) + return self.dumped_paths + + +def test_auto_resource_create(): + """CREATE a resource note from a new file (change=added).""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + app = await _make_app() + try: + vault_root = Path(app.config.vault_dir).absolute() + today = _today() + + print("\n" + "=" * 70) + print("[setup] vault_root =", vault_root) + print("[setup] today =", today) + print("=" * 70) + + file_path = _place_resource(vault_root, today, RESOURCE_FILENAME, RESOURCE_CONTENT_V1) + session_id = _compute_session_id(RESOURCE_FILENAME) + # daily_create prepends "session_agent_" to the session_id + expected_note_path = f"daily/{today}/session_agent_{session_id}.md" + + print(f"[CREATE] file_path = {file_path}") + print(f"[CREATE] session_id = {session_id}") + print(f"[CREATE] expected note= {expected_note_path}") + + with _AgentMemoryRecorder(DUMP_DIR, prefix="agent_resource_create") as recorder: + response = await app.run_job( + "auto_resource", + file_path=file_path, + change="added", + ) + dumped = await recorder.dump() + for p in dumped: + print(f"[CREATE] agent memory dumped: {p}") + + assert response.success is True, f"CREATE job failed: {response.answer!r}" + meta = response.metadata or {} + assert meta.get("path") == expected_note_path, f"Unexpected path: {meta!r}" + assert meta.get("action") == "added" + + note_path = vault_root / expected_note_path + assert note_path.is_file(), f"Created note not found at {note_path}" + + note_text = _read_text(note_path) + print("\n" + "=" * 70) + print(f"[CREATE] {note_path} ({len(note_text)} bytes)") + print(f"[CREATE] body:\n{note_text}") + print("=" * 70) + + topic_hits = [ + needle + for needle in ("v2.0", "July 15", "Alice", "Bob", "p99", "200ms", "Redis") + if needle in note_text + ] + print(f"[CREATE] landed topic facts: {topic_hits}") + assert ( + len(topic_hits) >= 3 + ), f"CREATE only captured {topic_hits!r} of expected facts\n--- CREATE ---\n{note_text}" + + print("\n" + "=" * 70) + print("test_auto_resource_create passed") + print("=" * 70) + finally: + await app.close() + + asyncio.run(run()) + + +def test_auto_resource_update(): + """UPDATE an existing resource note (change=modified).""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + app = await _make_app() + try: + vault_root = Path(app.config.vault_dir).absolute() + today = _today() + + print("\n" + "=" * 70) + print("[setup] vault_root =", vault_root) + print("[setup] today =", today) + print("=" * 70) + + # First create a note via "added" + file_path = _place_resource(vault_root, today, RESOURCE_FILENAME, RESOURCE_CONTENT_V1) + session_id = _compute_session_id(RESOURCE_FILENAME) + + response = await app.run_job("auto_resource", file_path=file_path, change="added") + assert response.success is True, f"Initial create failed: {response.answer!r}" + + note_abs = vault_root / "daily" / today / f"session_agent_{session_id}.md" + note_before = _read_text(note_abs) + print(f"[UPDATE] note before update ({len(note_before)} bytes)") + + # Now update the resource file and call with "modified" + _place_resource(vault_root, today, RESOURCE_FILENAME, RESOURCE_CONTENT_V2) + + with _AgentMemoryRecorder(DUMP_DIR, prefix="agent_resource_update") as recorder: + response = await app.run_job( + "auto_resource", + file_path=file_path, + change="modified", + ) + dumped = await recorder.dump() + for p in dumped: + print(f"[UPDATE] agent memory dumped: {p}") + + assert response.success is True, f"UPDATE job failed: {response.answer!r}" + meta = response.metadata or {} + assert meta.get("action") == "modified" + + note_after = _read_text(note_abs) + print("\n" + "=" * 70) + print(f"[UPDATE] {note_abs} ({len(note_before)} -> {len(note_after)} bytes)") + print(f"[UPDATE] body after:\n{note_after}") + print("=" * 70) + + new_hits = [ + needle + for needle in ("July 20", "150ms", "Dave", "rate limiting", "resolved") + if needle in note_after + ] + print(f"[UPDATE] landed new facts: {new_hits}") + assert ( + len(new_hits) >= 2 + ), f"UPDATE only landed {new_hits!r} of expected new facts\n--- AFTER ---\n{note_after}" + + print("\n" + "=" * 70) + print("test_auto_resource_update passed") + print("=" * 70) + finally: + await app.close() + + asyncio.run(run()) + + +def test_auto_resource_delete(): + """DELETE a resource note (change=deleted).""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + app = await _make_app() + try: + vault_root = Path(app.config.vault_dir).absolute() + today = _today() + + print("\n" + "=" * 70) + print("[setup] vault_root =", vault_root) + print("[setup] today =", today) + print("=" * 70) + + session_id = _compute_session_id(RESOURCE_FILENAME) + file_path = f"resource/{today}/{RESOURCE_FILENAME}" + + # Seed the note file (daily_create prepends "session_agent_") + note_filename = f"session_agent_{session_id}" + seed_body = "---\nname: test\ndescription: test note\n---\n\nSome content.\n" + note_path = _seed_resource_note(vault_root, today, note_filename, seed_body) + assert note_path.is_file() + print(f"[DELETE] seeded note: {note_path}") + + response = await app.run_job( + "auto_resource", + file_path=file_path, + change="deleted", + ) + + assert response.success is True, f"DELETE job failed: {response.answer!r}" + meta = response.metadata or {} + assert meta.get("action") == "deleted" + assert not note_path.is_file(), f"Note file still exists after delete: {note_path}" + + print(f"[DELETE] note removed: {note_path}") + print("\n" + "=" * 70) + print("test_auto_resource_delete passed") + print("=" * 70) + finally: + await app.close() + + asyncio.run(run()) + + +if __name__ == "__main__": + print("=== auto_resource integration test ===") + test_auto_resource_create() + test_auto_resource_update() + test_auto_resource_delete() + print("\nAll integration tests passed!") diff --git a/tests4/integration/test_stream_llm.py b/tests4/integration/test_stream_llm.py index 1afd4301..25a0b4bc 100644 --- a/tests4/integration/test_stream_llm.py +++ b/tests4/integration/test_stream_llm.py @@ -6,6 +6,7 @@ environment or a .env file at the repo root. Hits the real LLM API. import asyncio import os +import sys import tempfile from reme4 import Application @@ -14,6 +15,7 @@ from reme4.enumeration import ChunkEnum from reme4.schema import StreamChunk from reme4.steps.common.stream_llm_demo import StreamLLMDemoStep from reme4.utils import load_env +from reme4.utils.common_utils import execute_stream_task load_env() @@ -49,25 +51,32 @@ async def _test_stream_llm_basic_chat(): try: step = StreamLLMDemoStep(app_context=app.context) queue: asyncio.Queue = asyncio.Queue() + chunks: list[StreamChunk] = [] - response = await step( - stream_queue=queue, - query="What is 1 + 1? Reply with just the number.", + task = asyncio.create_task( + step( + stream_queue=queue, + query="Explain step by step how to compute 1 + 1, and give the final answer.", + ), ) - # Collect all chunks from the queue - chunks = [] - while not queue.empty(): - chunks.append(await queue.get()) + print("\n[stream_basic] streaming output:") + async for raw in execute_stream_task(queue, task, output_format="chunk"): + chunk: StreamChunk = raw # type: ignore[assignment] + chunks.append(chunk) + if chunk.chunk_type == ChunkEnum.CONTENT: + sys.stdout.write(chunk.chunk) + sys.stdout.flush() - # Should have received CONTENT chunks + response = task.result() + + # Should have received multiple CONTENT chunks for a longer response content_chunks = [c for c in chunks if c.chunk_type == ChunkEnum.CONTENT] - print(f"\n[stream_basic] got {len(content_chunks)} CONTENT chunks") - assert len(content_chunks) > 0, "Expected at least one CONTENT chunk" + print(f"\n\n[stream_basic] got {len(content_chunks)} CONTENT chunks") + assert len(content_chunks) > 1, "Expected multiple CONTENT chunks for streaming" # Final answer should be populated text = (response.answer or "").strip() - print(f"[stream_basic] final answer: {text!r}") assert text, "Empty assistant response" assert "2" in text, f"Expected '2' in response, got: {text!r}" @@ -86,24 +95,38 @@ async def _test_stream_llm_with_tool(): try: step = StreamLLMDemoStep(app_context=app.context) queue: asyncio.Queue = asyncio.Queue() + chunks: list[StreamChunk] = [] - response = await step( - stream_queue=queue, - query="Use the add tool to compute 21 + 21 and report the result.", - sys_prompt="Use the `add` tool whenever the user asks to add numbers.", - use_add_tool=True, + task = asyncio.create_task( + step( + stream_queue=queue, + query="Use the add tool to compute 21 + 21 and report the result.", + sys_prompt="Use the `add` tool whenever the user asks to add numbers.", + use_add_tool=True, + ), ) - # Collect all chunks - chunks = [] - while not queue.empty(): - chunks.append(await queue.get()) + print("\n[stream_tool] streaming output:") + async for raw in execute_stream_task(queue, task, output_format="chunk"): + chunk: StreamChunk = raw # type: ignore[assignment] + chunks.append(chunk) + if chunk.chunk_type == ChunkEnum.CONTENT: + sys.stdout.write(chunk.chunk) + sys.stdout.flush() + elif chunk.chunk_type == ChunkEnum.TOOL_CALL: + sys.stdout.write(f"\033[33m{chunk.chunk}\033[0m") + sys.stdout.flush() + elif chunk.chunk_type == ChunkEnum.TOOL_RESULT: + sys.stdout.write(f"\033[32m{chunk.chunk}\033[0m") + sys.stdout.flush() + + response = task.result() tool_call_chunks = [c for c in chunks if c.chunk_type == ChunkEnum.TOOL_CALL] tool_result_chunks = [c for c in chunks if c.chunk_type == ChunkEnum.TOOL_RESULT] content_chunks = [c for c in chunks if c.chunk_type == ChunkEnum.CONTENT] - print(f"\n[stream_tool] TOOL_CALL chunks: {len(tool_call_chunks)}") + print(f"\n\n[stream_tool] TOOL_CALL chunks: {len(tool_call_chunks)}") print(f"[stream_tool] TOOL_RESULT chunks: {len(tool_result_chunks)}") print(f"[stream_tool] CONTENT chunks: {len(content_chunks)}") @@ -119,16 +142,36 @@ async def _test_stream_llm_with_tool(): async def _test_stream_llm_fallback_no_stream(): - """Without stream_queue, falls back to non-streaming reply.""" + """Without stream_queue, still uses streaming under the hood for real-time output.""" with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): app = await _make_app() try: step = StreamLLMDemoStep(app_context=app.context) - response = await step( - query="What is 1 + 1? Reply with just the number.", + queue: asyncio.Queue = asyncio.Queue() + chunks: list[StreamChunk] = [] + + task = asyncio.create_task( + step( + stream_queue=queue, + query="Explain step by step how to compute 1 + 1, and give the final answer.", + ), ) + + print("\n[fallback_stream] streaming output:") + async for raw in execute_stream_task(queue, task, output_format="chunk"): + chunk: StreamChunk = raw # type: ignore[assignment] + chunks.append(chunk) + if chunk.chunk_type == ChunkEnum.CONTENT: + sys.stdout.write(chunk.chunk) + sys.stdout.flush() + elif chunk.chunk_type == ChunkEnum.THINK: + sys.stdout.write(f"\033[2m{chunk.chunk}\033[0m") + sys.stdout.flush() + + response = task.result() text = (response.answer or "").strip() - print(f"\n[fallback] response: {text!r}") + content_chunks = [c for c in chunks if c.chunk_type == ChunkEnum.CONTENT] + print(f"\n\n[fallback_stream] got {len(content_chunks)} CONTENT chunks") assert text, "Empty assistant response" assert "2" in text, f"Expected '2' in response, got: {text!r}" print("✓ test_stream_llm_fallback_no_stream passed") @@ -153,8 +196,6 @@ def test_stream_llm_fallback_no_stream(): async def _demo_stream_print(): """Real-time streaming print demo — ask a longer question to see chunked output.""" - import sys # pylint: disable=import-outside-toplevel,redefined-outer-name - with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): app = await _make_app() try: @@ -166,36 +207,29 @@ async def _demo_stream_print(): "Include the chain rule, gradient descent, and give a concrete example with numbers." ) - async def consumer(): - """Print chunks to terminal in real-time.""" - while True: - chunk = await queue.get() - if chunk.done: - break - if chunk.chunk_type == ChunkEnum.CONTENT: - sys.stdout.write(chunk.chunk) - sys.stdout.flush() - elif chunk.chunk_type == ChunkEnum.THINK: - sys.stdout.write(f"\033[2m{chunk.chunk}\033[0m") - sys.stdout.flush() - elif chunk.chunk_type == ChunkEnum.TOOL_CALL: - sys.stdout.write(f"\n\033[33m[tool_call] {chunk.chunk}\033[0m") - sys.stdout.flush() - elif chunk.chunk_type == ChunkEnum.TOOL_RESULT: - sys.stdout.write(f"\033[32m{chunk.chunk}\033[0m") - sys.stdout.flush() - print() - - consumer_task = asyncio.create_task(consumer()) - - await step( - stream_queue=queue, - query=query, - sys_prompt="You are a knowledgeable AI teacher. Explain concepts thoroughly.", + task = asyncio.create_task( + step( + stream_queue=queue, + query=query, + sys_prompt="You are a knowledgeable AI teacher. Explain concepts thoroughly.", + ), ) - # Signal done so consumer exits - await queue.put(StreamChunk(chunk_type=ChunkEnum.DONE, chunk="", done=True)) - await consumer_task + + async for raw in execute_stream_task(queue, task, output_format="chunk"): + chunk: StreamChunk = raw # type: ignore[assignment] + if chunk.chunk_type == ChunkEnum.CONTENT: + sys.stdout.write(chunk.chunk) + sys.stdout.flush() + elif chunk.chunk_type == ChunkEnum.THINK: + sys.stdout.write(f"\033[2m{chunk.chunk}\033[0m") + sys.stdout.flush() + elif chunk.chunk_type == ChunkEnum.TOOL_CALL: + sys.stdout.write(f"\n\033[33m[tool_call] {chunk.chunk}\033[0m") + sys.stdout.flush() + elif chunk.chunk_type == ChunkEnum.TOOL_RESULT: + sys.stdout.write(f"\033[32m{chunk.chunk}\033[0m") + sys.stdout.flush() + print() finally: await app.close() @@ -209,8 +243,6 @@ async def _run_all(): if __name__ == "__main__": - import sys - if len(sys.argv) > 1 and sys.argv[1] == "demo": asyncio.run(_demo_stream_print()) else: diff --git a/tests4/unit/test_auto_dream.py b/tests4/unit/test_auto_dream.py index 80e8398f..4752fb85 100644 --- a/tests4/unit/test_auto_dream.py +++ b/tests4/unit/test_auto_dream.py @@ -100,7 +100,7 @@ def test_scans_date_md_and_date_folder(): _touch(vault / "daily" / today / "session-a.md") _touch(vault / "daily" / today / "session-b.md") step = _make_step(vault, today) - ctx = RuntimeContext() + ctx = RuntimeContext(date=today) seen: list[str] = [] @@ -132,7 +132,7 @@ def test_resource_dir_is_not_scanned(): _touch(vault / "resource" / today / "spec.pdf") step = _make_step(vault, today) step.app_context.app_config.resource_dir = "resource" - ctx = RuntimeContext() + ctx = RuntimeContext(date=today) async def _fake_dream(rel, _hint): return DreamResult(used_llm=True, path=rel, summary="ok") @@ -158,7 +158,7 @@ def test_unchanged_files_skipped_via_catalog_mtime(): mtime = note.stat().st_mtime existing = [FileNode(path=f"daily/{today}/note.md", st_mtime=mtime)] step = _make_step(vault, today, existing_nodes=existing) - ctx = RuntimeContext() + ctx = RuntimeContext(date=today) with patch.object(step, "dream_one") as dream_mock: resp = await step(ctx) @@ -186,7 +186,7 @@ def test_changed_file_dreamed_and_catalog_updated(): stale = mtime - 999.0 existing = [FileNode(path=f"daily/{today}/note.md", st_mtime=stale)] step = _make_step(vault, today, existing_nodes=existing) - ctx = RuntimeContext() + ctx = RuntimeContext(date=today) async def _fake_dream(rel, _hint): return DreamResult(used_llm=True, path=rel, summary="ok") @@ -218,7 +218,7 @@ def test_deleted_file_dropped_from_catalog(): # No on-disk files for today; catalog has a stale entry for today. existing = [FileNode(path=f"daily/{today}/gone.md", st_mtime=123.0)] step = _make_step(vault, today, existing_nodes=existing) - ctx = RuntimeContext() + ctx = RuntimeContext(date=today) with patch.object(step, "dream_one") as dream_mock: resp = await step(ctx) @@ -246,7 +246,7 @@ def test_other_days_catalog_entries_untouched(): # Today: nothing on disk. Catalog has yesterday's entry. existing = [FileNode(path=f"daily/{yesterday}/note.md", st_mtime=99.0)] step = _make_step(vault, today, existing_nodes=existing) - ctx = RuntimeContext() + ctx = RuntimeContext(date=today) resp = await step(ctx) @@ -269,7 +269,7 @@ def test_failure_does_not_upsert(): today = "2026-06-04" _touch(vault / "daily" / today / "note.md") step = _make_step(vault, today) - ctx = RuntimeContext() + ctx = RuntimeContext(date=today) async def _fake_dream(rel, _hint): return DreamResult(used_llm=False, path=rel, error="boom") @@ -297,7 +297,7 @@ def test_phase1_empty_still_upserts(): note = _touch(vault / "daily" / today / "note.md") mtime = note.stat().st_mtime step = _make_step(vault, today) - ctx = RuntimeContext() + ctx = RuntimeContext(date=today) async def _fake_dream(rel, _hint): return DreamResult(used_llm=True, path=rel, skipped=True, summary="empty") @@ -325,7 +325,7 @@ def test_partial_failure_does_not_block_other_files(): _touch(vault / "daily" / today / "a.md") _touch(vault / "daily" / today / "b.md") step = _make_step(vault, today) - ctx = RuntimeContext() + ctx = RuntimeContext(date=today) async def _fake_dream(rel, _hint): if rel.endswith("a.md"): diff --git a/tests4/unit/test_background_steps.py b/tests4/unit/test_background_steps.py index ef329831..515945a0 100644 --- a/tests4/unit/test_background_steps.py +++ b/tests4/unit/test_background_steps.py @@ -1,15 +1,11 @@ -"""Tests for background steps: ScanStoreChangesStep + WatchChangesStep. +"""Tests for background steps: scan/watch/dispatch steps. -Both steps are subclasses of BaseStep. To exercise them without spinning up the -full ApplicationContext, we pass real (started) file_store/file_chunker via the -step's kwargs (so the BaseStep _resolve() machinery returns them). +Both scan steps are subclasses of BaseStep. To exercise them without spinning up +the full ApplicationContext, we pass real (started) file_store/file_chunker via +the step's kwargs (so the BaseStep _resolve() machinery returns them). -ScanStoreChangesStep writes its result into ``context["changes"]`` for a downstream -``update_index_step`` to consume; tests assert against that key directly. - -The catalog-side sibling (ScanCatalogChangesStep) shares the same diff helper -and is exercised through the dream-loop integration tests; covering it here -would duplicate the file_store-diff assertions without adding signal. +ScanStoreChangesStep writes its result into ``context["changes"]`` for a +downstream ``update_index_step`` to consume; tests assert against that key. """ # pylint: disable=protected-access @@ -19,13 +15,15 @@ import os import tempfile import warnings from pathlib import Path +from unittest.mock import AsyncMock, MagicMock from watchfiles import Change from reme4.components.file_chunker import DefaultFileChunker from reme4.components.file_store import LocalFileStore from reme4.components.runtime_context import RuntimeContext -from reme4.steps import ScanStoreChangesStep, WatchChangesStep +from reme4.steps import ForeachDispatchStep, LogChangesStep, ScanStoreChangesStep, WatchChangesStep +from reme4.steps.index._watch_rules import WatchRule, build_watch_rules, collect_existing, match_file warnings.filterwarnings("ignore", category=DeprecationWarning, module="jieba") warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources") @@ -54,140 +52,249 @@ def write_file(path: Path, content: str = "x") -> Path: return path +def _make_app_context(vault_path: Path, daily_dir="daily", digest_dir="digest", resource_dir="resource"): + """Create a mock app_context with app_config pointing to the given vault.""" + ctx = MagicMock() + ctx.app_config.vault_dir = str(vault_path) + ctx.app_config.daily_dir = daily_dir + ctx.app_config.digest_dir = digest_dir + ctx.app_config.resource_dir = resource_dir + return ctx + + +# --------------------------------------------------------------------------- +# _watch_rules module tests +# --------------------------------------------------------------------------- + + +def test_build_watch_rules_basic(): + """Build rules from watch_dirs and watch_suffixes.""" + app_config = MagicMock() + app_config.daily_dir = "daily" + app_config.digest_dir = "digest" + app_config.resource_dir = "resource" + vault = Path("/fake/vault") + + rules = build_watch_rules(app_config, vault, watch_dirs=["daily_dir", "digest_dir"], watch_suffixes=["md"]) + assert len(rules) == 2 + assert rules[0].path == vault / "daily" + assert rules[0].suffixes == ["md"] + assert rules[1].path == vault / "digest" + print("✓ test_build_watch_rules_basic passed") + + +def test_build_watch_rules_multiple_suffixes(): + """Multiple suffixes are forwarded to each rule.""" + app_config = MagicMock() + app_config.daily_dir = "daily" + app_config.resource_dir = "resource" + vault = Path("/fake/vault") + + rules = build_watch_rules( + app_config, + vault, + watch_dirs=["daily_dir", "resource_dir"], + watch_suffixes=["md", "jsonl"], + ) + assert len(rules) == 2 + assert rules[0].suffixes == ["md", "jsonl"] + assert rules[1].suffixes == ["md", "jsonl"] + print("✓ test_build_watch_rules_multiple_suffixes passed") + + +def test_build_watch_rules_fallback_literal(): + """Unknown field names are used as literal directory names.""" + app_config = MagicMock(spec=[]) # no attributes + vault = Path("/fake/vault") + rules = build_watch_rules(app_config, vault, watch_dirs=["custom_dir"], watch_suffixes=["txt"]) + assert rules[0].path == vault / "custom_dir" + print("✓ test_build_watch_rules_fallback_literal passed") + + +def test_match_file_suffix(): + """match_file accepts files matching suffix under rule path.""" + rules = [WatchRule(path=Path("/vault/daily"), suffixes=["md"])] + assert match_file("/vault/daily/2026-01-01.md", rules) + assert match_file("/vault/daily/sub/note.md", rules) + assert not match_file("/vault/daily/file.txt", rules) + assert not match_file("/vault/other/file.md", rules) + print("✓ test_match_file_suffix passed") + + +def test_match_file_no_suffix_filter(): + """Empty suffixes list means all files match.""" + rules = [WatchRule(path=Path("/vault/resource"), suffixes=[])] + assert match_file("/vault/resource/anything.xyz", rules) + assert match_file("/vault/resource/sub/deep.pdf", rules) + assert not match_file("/vault/other/file.md", rules) + print("✓ test_match_file_no_suffix_filter passed") + + +def test_collect_existing_filters(): + """collect_existing applies suffix rules correctly.""" + with tempfile.TemporaryDirectory() as tmpdir: + vault = Path(tmpdir) + daily = vault / "daily" + resource = vault / "resource" + write_file(daily / "note.md") + write_file(daily / "ignore.txt") + write_file(resource / "data.json") + write_file(resource / "binary.png") + + rules = [ + WatchRule(path=daily, suffixes=["md"]), + WatchRule(path=resource, suffixes=["json"]), + ] + result = collect_existing(rules, recursive=True) + paths = set(result.keys()) + assert str((daily / "note.md").absolute()) in paths + assert str((daily / "ignore.txt").absolute()) not in paths + assert str((resource / "data.json").absolute()) in paths + assert str((resource / "binary.png").absolute()) not in paths + print("✓ test_collect_existing_filters passed") + + # --------------------------------------------------------------------------- # ScanStoreChangesStep # --------------------------------------------------------------------------- -async def _make_scan_step( - watch_paths: list[str] | str = "vault", - suffix_filters: list[str] | None = None, - recursive: bool = True, -) -> tuple[ScanStoreChangesStep, RuntimeContext, LocalFileStore, DefaultFileChunker]: +async def _make_scan_step(vault_path: Path, watch_dirs=None, watch_suffixes=None, recursive=True): fs = LocalFileStore(name="test_store", embedding_store="") - parser = DefaultFileChunker() + chunker = DefaultFileChunker() await fs.start() - await parser.start() - step = ScanStoreChangesStep( - recursive=recursive, - file_store=fs, - file_chunker=parser, - ) + await chunker.start() + app_ctx = _make_app_context(vault_path) + step = ScanStoreChangesStep(recursive=recursive, file_store=fs, file_chunker=chunker, app_context=app_ctx) context = RuntimeContext( - watch_paths=watch_paths, - suffix_filters=suffix_filters or ["md"], + watch_dirs=watch_dirs or ["daily_dir", "digest_dir"], + watch_suffixes=watch_suffixes or ["md"], ) - return step, context, fs, parser + return step, context, fs, chunker -async def _teardown(fs: LocalFileStore, parser: DefaultFileChunker) -> None: - await parser.close() +async def _teardown(fs: LocalFileStore, chunker: DefaultFileChunker) -> None: + await chunker.close() await fs.close() def test_scan_changes_initial_all_added(): - """First run on a fresh store emits 'added' for every existing file (abs paths).""" + """First run on a fresh store emits 'added' for every existing file.""" async def run(): with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): - # Use Path.cwd() as the basis so we match BaseStep.vault_path on macOS - # (where /var resolves to /private/var via a symlink). cwd = Path.cwd() - vault = cwd / "vault" - write_file(vault / "a.md", "alpha") - write_file(vault / "b.md", "beta") - step, ctx, fs, parser = await _make_scan_step() + write_file(cwd / "daily" / "a.md", "alpha") + write_file(cwd / "daily" / "b.md", "beta") + (cwd / "digest").mkdir(parents=True, exist_ok=True) + + step, ctx, fs, chunker = await _make_scan_step(cwd) try: resp = await step(ctx) counts = resp.metadata["counts"] assert counts == {"added": 2, "modified": 0, "deleted": 0} - changes = ctx["changes"] - kinds = sorted(item["change"] for item in changes) - paths = sorted(item["path"] for item in changes) - assert kinds == ["added", "added"] - expected = sorted([str(cwd / "vault/a.md"), str(cwd / "vault/b.md")]) - assert paths == expected + assert len(ctx["changes"]) == 2 finally: - await _teardown(fs, parser) + await _teardown(fs, chunker) print("✓ test_scan_changes_initial_all_added passed") asyncio.run(run()) -def test_scan_changes_no_changes_emits_empty_list(): - """A second run over an unchanged store reports zero counts and empty changes.""" +def test_scan_changes_no_changes(): + """Second run over an unchanged store reports zero counts.""" async def run(): with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): cwd = Path.cwd() - vault = cwd / "vault" - a = write_file(vault / "a.md", "alpha") - step, ctx, fs, parser = await _make_scan_step() + a = write_file(cwd / "daily" / "a.md", "alpha") + (cwd / "digest").mkdir(parents=True, exist_ok=True) + + step, ctx, fs, chunker = await _make_scan_step(cwd) try: - node, chunks = await parser.parse(a) + node, chunks = await chunker.chunk(a) await fs.upsert([(node, chunks)]) - - resp = await step(ctx) - counts = resp.metadata["counts"] - assert counts == {"added": 0, "modified": 0, "deleted": 0} - assert ctx["changes"] == [] - finally: - await _teardown(fs, parser) - print("✓ test_scan_changes_no_changes_emits_empty_list passed") - - asyncio.run(run()) - - -def test_scan_changes_detects_modify_and_delete(): - """Second pass distinguishes added/modified/deleted; paths are absolute.""" - - async def run(): - with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): - cwd = Path.cwd() - vault = cwd / "vault" - a = write_file(vault / "a.md", "alpha") - b = write_file(vault / "b.md", "beta") - step, ctx, fs, parser = await _make_scan_step() - try: - # Seed via direct parse/upsert. - for p in (a, b): - node, chunks = await parser.parse(p) - await fs.upsert([(node, chunks)]) - - # Modify a, delete b, add c. - a.write_text("alpha-v2", encoding="utf-8") - os.utime(a, (9_999_999_999, 9_999_999_999)) - b.unlink() - c = write_file(vault / "c.md", "gamma") - - resp = await step(ctx) - counts = resp.metadata["counts"] - assert counts == {"added": 1, "modified": 1, "deleted": 1} - by_kind = {item["change"]: item["path"] for item in ctx["changes"]} - assert by_kind["added"] == str(c) - assert by_kind["modified"] == str(a) - assert by_kind["deleted"] == str(b) - finally: - await _teardown(fs, parser) - print("✓ test_scan_changes_detects_modify_and_delete passed") - - asyncio.run(run()) - - -def test_scan_changes_missing_watch_path_silently_skipped(): - """Non-existent watch_paths entries are dropped silently.""" - - async def run(): - with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): - (Path(tmpdir) / "vault").mkdir() - step, ctx, fs, parser = await _make_scan_step(watch_paths=["vault", "ghost"]) - try: resp = await step(ctx) assert resp.metadata["counts"] == {"added": 0, "modified": 0, "deleted": 0} assert ctx["changes"] == [] finally: - await _teardown(fs, parser) - print("✓ test_scan_changes_missing_watch_path_silently_skipped passed") + await _teardown(fs, chunker) + print("✓ test_scan_changes_no_changes passed") + + asyncio.run(run()) + + +def test_scan_changes_detect_modify_delete(): + """Second pass distinguishes added/modified/deleted.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): + cwd = Path.cwd() + a = write_file(cwd / "daily" / "a.md", "alpha") + b = write_file(cwd / "daily" / "b.md", "beta") + (cwd / "digest").mkdir(parents=True, exist_ok=True) + + step, ctx, fs, chunker = await _make_scan_step(cwd) + try: + for p in (a, b): + node, chunks = await chunker.chunk(p) + await fs.upsert([(node, chunks)]) + a.write_text("alpha-v2", encoding="utf-8") + os.utime(a, (9_999_999_999, 9_999_999_999)) + b.unlink() + write_file(cwd / "daily" / "c.md", "gamma") + + resp = await step(ctx) + counts = resp.metadata["counts"] + assert counts == {"added": 1, "modified": 1, "deleted": 1} + finally: + await _teardown(fs, chunker) + print("✓ test_scan_changes_detect_modify_delete passed") + + asyncio.run(run()) + + +def test_scan_changes_missing_dir_skipped(): + """Non-existent watch_dirs entries are dropped silently.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): + cwd = Path.cwd() + (cwd / "daily").mkdir() + # digest dir missing + step, ctx, fs, chunker = await _make_scan_step(cwd) + try: + resp = await step(ctx) + assert resp.metadata["counts"] == {"added": 0, "modified": 0, "deleted": 0} + finally: + await _teardown(fs, chunker) + print("✓ test_scan_changes_missing_dir_skipped passed") + + asyncio.run(run()) + + +def test_scan_changes_resource_dir(): + """Scanning resource_dir with multiple suffixes works.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): + cwd = Path.cwd() + resource = cwd / "resource" + write_file(resource / "data.json", "{}") + write_file(resource / "note.md", "# Note") + write_file(resource / "image.png", "binary") + + step, ctx, fs, chunker = await _make_scan_step( + cwd, + watch_dirs=["resource_dir"], + watch_suffixes=["md", "json"], + ) + try: + resp = await step(ctx) + assert resp.metadata["counts"]["added"] == 2 + finally: + await _teardown(fs, chunker) + print("✓ test_scan_changes_resource_dir passed") asyncio.run(run()) @@ -201,62 +308,203 @@ def test_watch_changes_requires_stop_event(): """Missing stop_event in context raises a clear error.""" async def run(): - step = WatchChangesStep() - step.context = RuntimeContext(watch_paths=["vault"], suffix_filters=["md"]) - try: - await step.execute() - except RuntimeError as e: - assert "stop_event" in str(e) - else: - raise AssertionError("expected RuntimeError") + with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): + cwd = Path.cwd() + (cwd / "daily").mkdir() + app_ctx = _make_app_context(cwd) + step = WatchChangesStep(app_context=app_ctx) + step.context = RuntimeContext(watch_dirs=["daily_dir"], watch_suffixes=["md"]) + try: + await step.execute() + except RuntimeError as e: + assert "stop_event" in str(e) + else: + raise AssertionError("expected RuntimeError") print("✓ test_watch_changes_requires_stop_event passed") asyncio.run(run()) -def test_watch_changes_raises_when_no_valid_paths(): - """With no valid watch_paths, the step raises so the BackgroundJob supervisor can back off.""" +def test_watch_changes_raises_no_valid_paths(): + """With no valid watch_paths, the step raises.""" async def run(): with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): - step = WatchChangesStep() + cwd = Path.cwd() + app_ctx = _make_app_context(cwd) + step = WatchChangesStep(app_context=app_ctx) stop = asyncio.Event() - step.context = RuntimeContext( - stop_event=stop, - watch_paths=["ghost"], - suffix_filters=["md"], - ) - + step.context = RuntimeContext(stop_event=stop, watch_dirs=["daily_dir"], watch_suffixes=["md"]) try: await step.execute() except RuntimeError as e: assert "No valid watch paths" in str(e) else: raise AssertionError("expected RuntimeError") - print("✓ test_watch_changes_raises_when_no_valid_paths passed") + print("✓ test_watch_changes_raises_no_valid_paths passed") asyncio.run(run()) -def test_watch_changes_filter_only_passes_md(): - """The internal filter pulls suffix_filters from runtime context.""" +def test_watch_changes_filter_matches_rules(): + """The internal filter uses watch rules from context.""" + with tempfile.TemporaryDirectory() as tmpdir: + vault = Path(tmpdir) + (vault / "daily").mkdir() + (vault / "digest").mkdir() + (vault / "resource").mkdir() + app_ctx = _make_app_context(vault) - step = WatchChangesStep() - step.context = RuntimeContext(suffix_filters=["md"]) - assert step._filter(Change.added, "/x/foo.md") - assert not step._filter(Change.added, "/x/foo.txt") - print("✓ test_watch_changes_filter_only_passes_md passed") + step = WatchChangesStep(app_context=app_ctx) + step.context = RuntimeContext(watch_dirs=["daily_dir", "digest_dir"], watch_suffixes=["md"]) + step._rules = step._get_watch_rules() + + assert step._filter(Change.added, str(vault / "daily/foo.md")) + assert step._filter(Change.added, str(vault / "digest/bar.md")) + assert not step._filter(Change.added, str(vault / "daily/foo.txt")) + assert not step._filter(Change.added, str(vault / "resource/file.md")) + + print("✓ test_watch_changes_filter_matches_rules passed") + + +def test_watch_changes_dispatch_steps_list(): + """dispatch_steps config properly merges dispatch_step and dispatch_steps.""" + step1 = WatchChangesStep(dispatch_step="update_index_step") + assert step1.dispatch_steps == ["update_index_step"] + + step2 = WatchChangesStep(dispatch_steps=["update_catalog_step", "foreach_dispatch_step"]) + assert step2.dispatch_steps == ["update_catalog_step", "foreach_dispatch_step"] + + step3 = WatchChangesStep(dispatch_step="a", dispatch_steps=["b", "c"]) + assert step3.dispatch_steps == ["b", "c"] # dispatch_steps takes priority + + print("✓ test_watch_changes_dispatch_steps_list passed") + + +# --------------------------------------------------------------------------- +# ForeachDispatchStep +# --------------------------------------------------------------------------- + + +def test_foreach_dispatch_no_job(): + """Without dispatch_job, step logs warning and returns success.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): + cwd = Path.cwd() + app_ctx = _make_app_context(cwd) + step = ForeachDispatchStep(app_context=app_ctx) + ctx = RuntimeContext(changes=[{"change": "added", "path": "/x/y.md"}]) + resp = await step(ctx) + assert resp.success is True + assert resp.metadata.get("dispatched") is None # skipped early + print("✓ test_foreach_dispatch_no_job passed") + + asyncio.run(run()) + + +def test_foreach_dispatch_calls_job(): + """ForeachDispatchStep calls run_job for each change.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): + cwd = Path.cwd() + app_ctx = _make_app_context(cwd) + step = ForeachDispatchStep(app_context=app_ctx) + changes = [ + {"change": "added", "path": str(cwd / "resource/2026-01-01/file.md")}, + {"change": "modified", "path": str(cwd / "resource/2026-01-01/data.json")}, + ] + ctx = RuntimeContext(changes=changes, dispatch_job="auto_resource") + + mock_job = AsyncMock() + app_ctx.jobs = {"auto_resource": mock_job} + resp = await step(ctx) + assert resp.success is True + assert resp.metadata["dispatched"] == 2 + assert mock_job.call_count == 2 + # Verify vault-relative paths were passed + calls = mock_job.call_args_list + assert calls[0].kwargs["file_path"] == "resource/2026-01-01/file.md" + assert calls[0].kwargs["change"] == "added" + assert calls[1].kwargs["file_path"] == "resource/2026-01-01/data.json" + assert calls[1].kwargs["change"] == "modified" + print("✓ test_foreach_dispatch_calls_job passed") + + asyncio.run(run()) + + +def test_foreach_dispatch_handles_error(): + """ForeachDispatchStep continues on job failure.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): + cwd = Path.cwd() + app_ctx = _make_app_context(cwd) + step = ForeachDispatchStep(app_context=app_ctx) + changes = [ + {"change": "added", "path": str(cwd / "resource/a.md")}, + {"change": "added", "path": str(cwd / "resource/b.md")}, + ] + ctx = RuntimeContext(changes=changes, dispatch_job="failing_job") + + mock_job = AsyncMock(side_effect=RuntimeError("boom")) + app_ctx.jobs = {"failing_job": mock_job} + resp = await step(ctx) + assert resp.success is True # still succeeds + assert mock_job.call_count == 2 # tried both + print("✓ test_foreach_dispatch_handles_error passed") + + asyncio.run(run()) + + +# --------------------------------------------------------------------------- +# LogChangesStep +# --------------------------------------------------------------------------- + + +def test_log_changes_step(): + """LogChangesStep logs and reports count.""" + + async def run(): + step = LogChangesStep() + changes = [ + {"change": "added", "path": "/vault/daily/note.md"}, + {"change": "deleted", "path": "/vault/daily/old.md"}, + ] + ctx = RuntimeContext(changes=changes) + resp = await step(ctx) + assert resp.success is True + assert resp.metadata["count"] == 2 + print("✓ test_log_changes_step passed") + + asyncio.run(run()) if __name__ == "__main__": print("\n=== Background Steps Tests ===") + # _watch_rules + test_build_watch_rules_basic() + test_build_watch_rules_multiple_suffixes() + test_build_watch_rules_fallback_literal() + test_match_file_suffix() + test_match_file_no_suffix_filter() + test_collect_existing_filters() # ScanStoreChangesStep test_scan_changes_initial_all_added() - test_scan_changes_no_changes_emits_empty_list() - test_scan_changes_detects_modify_and_delete() - test_scan_changes_missing_watch_path_silently_skipped() + test_scan_changes_no_changes() + test_scan_changes_detect_modify_delete() + test_scan_changes_missing_dir_skipped() + test_scan_changes_resource_dir() # WatchChangesStep test_watch_changes_requires_stop_event() - test_watch_changes_raises_when_no_valid_paths() - test_watch_changes_filter_only_passes_md() + test_watch_changes_raises_no_valid_paths() + test_watch_changes_filter_matches_rules() + test_watch_changes_dispatch_steps_list() + # ForeachDispatchStep + test_foreach_dispatch_no_job() + test_foreach_dispatch_calls_job() + test_foreach_dispatch_handles_error() + # LogChangesStep + test_log_changes_step() print("\n所有测试通过!") diff --git a/tests4/unit/test_channel_notify.py b/tests4/unit/test_channel_notify.py index c8a8e689..69a62354 100644 --- a/tests4/unit/test_channel_notify.py +++ b/tests4/unit/test_channel_notify.py @@ -6,7 +6,7 @@ from pathlib import Path from reme4.components.application_context import ApplicationContext from reme4.components.service.mcp_service import ChannelSink from reme4.components.runtime_context import RuntimeContext -from reme4.steps.index.channel_notify import ChannelNotifyStep +from reme4.steps.channel.channel_notify import ChannelNotifyStep class _StubSession: diff --git a/tests4/unit/test_daily_steps.py b/tests4/unit/test_daily_steps.py index f40c8adf..c950b077 100644 --- a/tests4/unit/test_daily_steps.py +++ b/tests4/unit/test_daily_steps.py @@ -79,7 +79,7 @@ async def _make_store_with_dailies(entries: list[tuple[str, str, str]]) -> Local day_dir = Path.cwd() / "daily" / day day_dir.mkdir(parents=True, exist_ok=True) text = f"---\nname: {session_id}\n---\n{body}\n" - (day_dir / f"{session_id}.md").write_text(text, encoding="utf-8") + (day_dir / f"session_agent_{session_id}.md").write_text(text, encoding="utf-8") return store @@ -95,7 +95,7 @@ async def _seed_note(date: str, session_id: str, name: str = "", description: st if description: fm_lines.append(f"description: {description}") text = "---\n" + "\n".join(fm_lines) + "\n---\nbody\n" - (day_dir / f"{session_id}.md").write_text(text, encoding="utf-8") + (day_dir / f"session_agent_{session_id}.md").write_text(text, encoding="utf-8") # -- daily_list_step ---------------------------------------------------------- @@ -119,8 +119,8 @@ def test_daily_list_default_date_is_today(): assert payload["date"] == _today() assert payload["count"] == 2 answer = step.context.response.answer - assert f"daily/{_today()}/today-a.md" in answer - assert f"daily/{_today()}/today-b.md" in answer + assert f"daily/{_today()}/session_agent_today-a.md" in answer + assert f"daily/{_today()}/session_agent_today-b.md" in answer await store.close() print("✓ test_daily_list_default_date_is_today passed") @@ -144,7 +144,7 @@ def test_daily_list_filters_by_date(): assert payload["date"] == "2026-05-18" assert payload["count"] == 1 answer = step.context.response.answer - assert "daily/2026-05-18/a.md" in answer + assert "daily/2026-05-18/session_agent_a.md" in answer await store.close() print("✓ test_daily_list_filters_by_date passed") @@ -169,7 +169,7 @@ def test_daily_list_returns_path_session_id_metadata(): payload = _metadata(step) assert payload["count"] == 1 answer = step.context.response.answer - assert "daily/2026-05-18/alpha.md" in answer + assert "daily/2026-05-18/session_agent_alpha.md" in answer assert "Alpha Project" in answer assert "JWT auth migration" in answer await store.close() @@ -200,7 +200,7 @@ def test_daily_list_ignores_subdirectories(): payload = _metadata(step) assert payload["count"] == 1 answer = step.context.response.answer - assert "daily/2026-05-18/main.md" in answer + assert "daily/2026-05-18/session_agent_main.md" in answer await store.close() print("✓ test_daily_list_ignores_subdirectories passed") @@ -285,9 +285,9 @@ def test_daily_create_provisions_note_and_refreshes_index(): assert payload["created"] is True assert payload["date"] == "2026-05-18" assert payload["session_id"] == "kickoff" - assert payload["path"] == "daily/2026-05-18/kickoff.md" + assert payload["path"] == "daily/2026-05-18/session_agent_kickoff.md" - note = Path(tmp) / "daily" / "2026-05-18" / "kickoff.md" + note = Path(tmp) / "daily" / "2026-05-18" / "session_agent_kickoff.md" text = note.read_text(encoding="utf-8") assert "name: kickoff" in text # Body is empty — file is frontmatter + trailing newline. @@ -295,7 +295,7 @@ def test_daily_create_provisions_note_and_refreshes_index(): index = Path(tmp) / "daily" / "2026-05-18.md" assert index.is_file() - assert "[[daily/2026-05-18/kickoff.md]]" in index.read_text(encoding="utf-8") + assert "[[daily/2026-05-18/session_agent_kickoff.md]]" in index.read_text(encoding="utf-8") await store.close() print("✓ test_daily_create_provisions_note_and_refreshes_index passed") @@ -310,7 +310,7 @@ def test_daily_create_is_idempotent_on_existing(): store = await _make_store_with_dailies( [("2026-05-18", "ongoing", "old body")], ) - file_path = Path(tmp) / "daily" / "2026-05-18" / "ongoing.md" + file_path = Path(tmp) / "daily" / "2026-05-18" / "session_agent_ongoing.md" before = file_path.read_text(encoding="utf-8") step = daily_create_step.DailyCreateStep(file_store=store) @@ -319,7 +319,7 @@ def test_daily_create_is_idempotent_on_existing(): assert step.context.response.success is True assert payload["created"] is False - assert payload["path"] == "daily/2026-05-18/ongoing.md" + assert payload["path"] == "daily/2026-05-18/session_agent_ongoing.md" assert file_path.read_text(encoding="utf-8") == before assert payload["index"]["path"] == "daily/2026-05-18.md" await store.close() @@ -338,7 +338,7 @@ def test_daily_create_default_date_is_today(): await step(session_id="today-task") payload = _metadata(step) assert payload["date"] == _today() - assert payload["path"] == f"daily/{_today()}/today-task.md" + assert payload["path"] == f"daily/{_today()}/session_agent_today-task.md" assert payload["created"] is True await store.close() print("✓ test_daily_create_default_date_is_today passed") @@ -355,7 +355,7 @@ def test_daily_create_default_frontmatter_uses_session_id_as_name(): step = daily_create_step.DailyCreateStep(file_store=store) await step(session_id="auth-refactor", date="2026-05-18") - note = Path(tmp) / "daily" / "2026-05-18" / "auth-refactor.md" + note = Path(tmp) / "daily" / "2026-05-18" / "session_agent_auth-refactor.md" text = note.read_text(encoding="utf-8") assert "name: auth-refactor" in text assert "description:" in text @@ -414,7 +414,7 @@ def test_daily_create_then_skip_round_trip(): first = _metadata(step) assert first["created"] is True - note = Path(tmp) / "daily" / "2026-05-18" / "probe.md" + note = Path(tmp) / "daily" / "2026-05-18" / "session_agent_probe.md" before = note.read_text(encoding="utf-8") await step(session_id="probe", date="2026-05-18") @@ -446,8 +446,8 @@ def test_day_index_lists_each_note(): await daily_reindex_step.DailyReindexStep(file_store=store)(date="2026-05-18") text = _day_index_text(tmp, "2026-05-18") - assert "[[daily/2026-05-18/alpha.md]]" in text - assert "[[daily/2026-05-18/beta.md]]" in text + assert "[[daily/2026-05-18/session_agent_alpha.md]]" in text + assert "[[daily/2026-05-18/session_agent_beta.md]]" in text assert "Alpha Project" in text assert "Beta Project" in text await store.close() @@ -474,13 +474,19 @@ def test_day_index_includes_note_descriptions(): await daily_reindex_step.DailyReindexStep(file_store=store)(date="2026-05-18") text = _day_index_text(tmp, "2026-05-18") # name + description inline on the same line as the wikilink - assert "[[daily/2026-05-18/alpha.md]] name: Alpha Project description: 实现 JWT auth 中间件" in text - assert "[[daily/2026-05-18/beta.md]] name: beta description: 调研增值税新政对 SaaS 的影响" in text - # gamma has no description → only name is emitted, no trailing `description:` cruft - assert "[[daily/2026-05-18/gamma.md]] name: Gamma\n" in text or text.rstrip().endswith( - "[[daily/2026-05-18/gamma.md]] name: Gamma", + assert ( + "[[daily/2026-05-18/session_agent_alpha.md]] name: Alpha Project description: 实现 JWT auth 中间件" + in text ) - assert "description:" not in text.split("[[daily/2026-05-18/gamma.md]]")[1].split("\n")[0] + assert ( + "[[daily/2026-05-18/session_agent_beta.md]] name: beta description: 调研增值税新政对 SaaS 的影响" + in text + ) + # gamma has no description → only name is emitted, no trailing `description:` cruft + assert "[[daily/2026-05-18/session_agent_gamma.md]] name: Gamma\n" in text or text.rstrip().endswith( + "[[daily/2026-05-18/session_agent_gamma.md]] name: Gamma", + ) + assert "description:" not in text.split("[[daily/2026-05-18/session_agent_gamma.md]]")[1].split("\n")[0] await store.close() print("✓ test_day_index_includes_note_descriptions passed") @@ -534,7 +540,7 @@ def test_day_index_preserves_user_content_outside_marker(): assert "MY HAND-WRITTEN NOTE" in after assert "这是我手写的备忘" in after assert "## 我的笔记" in after - assert "[[daily/2026-05-18/beta.md]]" in after + assert "[[daily/2026-05-18/session_agent_beta.md]]" in after await store.close() print("✓ test_day_index_preserves_user_content_outside_marker passed") @@ -568,8 +574,8 @@ def test_daily_reindex_returns_write_view(): assert payload["notes_count"] == 2 text = _day_index_text(tmp, "2026-05-18") - assert "[[daily/2026-05-18/alpha.md]]" in text - assert "[[daily/2026-05-18/beta.md]]" in text + assert "[[daily/2026-05-18/session_agent_alpha.md]]" in text + assert "[[daily/2026-05-18/session_agent_beta.md]]" in text await store.close() print("✓ test_daily_reindex_returns_write_view passed") diff --git a/tests4/unit/test_default_file_chunker.py b/tests4/unit/test_default_file_chunker.py index 7c46526c..34672367 100644 --- a/tests4/unit/test_default_file_chunker.py +++ b/tests4/unit/test_default_file_chunker.py @@ -19,8 +19,8 @@ def test_parse_empty_file(): temp_path = f.name try: - parser = DefaultFileChunker() - file_node, chunks = await parser.parse(temp_path) + chunker = DefaultFileChunker() + file_node, chunks = await chunker.chunk(temp_path) assert file_node.path == temp_path assert len(chunks) == 0 print("✓ test_parse_empty_file passed") @@ -40,8 +40,8 @@ def test_parse_small_file(): temp_path = f.name try: - parser = DefaultFileChunker(chunk_byte_size=10000) - _, chunks = await parser.parse(temp_path) + chunker = DefaultFileChunker(chunk_byte_size=10000) + _, chunks = await chunker.chunk(temp_path) assert len(chunks) == 1 assert chunks[0].start_line == 1 assert chunks[0].end_line == 3 @@ -64,8 +64,8 @@ def test_parse_multiline_file(): temp_path = f.name try: - parser = DefaultFileChunker(chunk_byte_size=10000) - _, chunks = await parser.parse(temp_path) + chunker = DefaultFileChunker(chunk_byte_size=10000) + _, chunks = await chunker.chunk(temp_path) assert len(chunks) == 1 assert chunks[0].start_line == 1 assert chunks[0].end_line == 5 @@ -88,8 +88,8 @@ def test_parse_chunked_file(): temp_path = f.name try: - parser = DefaultFileChunker(chunk_byte_size=5000, overlap_byte_size=100) - _, chunks = await parser.parse(temp_path) + chunker = DefaultFileChunker(chunk_byte_size=5000, overlap_byte_size=100) + _, chunks = await chunker.chunk(temp_path) assert len(chunks) > 1, f"Expected multiple chunks, got {len(chunks)}" # Verify overlap by checking that consecutive chunks share some content print(f" Created {len(chunks)} chunks") @@ -110,8 +110,8 @@ def test_parse_with_custom_encoding(): temp_path = f.name try: - parser = DefaultFileChunker(encoding="utf-8") - _, chunks = await parser.parse(temp_path) + chunker = DefaultFileChunker(encoding="utf-8") + _, chunks = await chunker.chunk(temp_path) assert len(chunks) >= 1 assert "你好世界" in chunks[0].text print("✓ test_parse_with_custom_encoding passed") @@ -131,8 +131,8 @@ def test_file_node_properties(): temp_path = f.name try: - parser = DefaultFileChunker() - file_node, _ = await parser.parse(temp_path) + chunker = DefaultFileChunker() + file_node, _ = await chunker.chunk(temp_path) assert hasattr(file_node, "path") assert hasattr(file_node, "st_mtime") assert file_node.st_mtime > 0 @@ -153,8 +153,8 @@ def test_file_chunk_properties(): temp_path = f.name try: - parser = DefaultFileChunker() - _, chunks = await parser.parse(temp_path) + chunker = DefaultFileChunker() + _, chunks = await chunker.chunk(temp_path) chunk = chunks[0] assert hasattr(chunk, "path") assert hasattr(chunk, "start_line") @@ -298,8 +298,8 @@ def test_parse_links_in_file(): temp_path = f.name try: - parser = DefaultFileChunker() - file_node, _ = await parser.parse(temp_path) + chunker = DefaultFileChunker() + file_node, _ = await chunker.chunk(temp_path) triples = {(link.predicate, link.target_path, link.target_anchor) for link in file_node.links} assert (None, "alpha", None) in triples assert (None, "beta", "h2") in triples @@ -326,9 +326,9 @@ def test_parse_links_empty_when_no_content(): fm_only_path = f.name try: - parser = DefaultFileChunker() - node1, _ = await parser.parse(empty_path) - node2, _ = await parser.parse(fm_only_path) + chunker = DefaultFileChunker() + node1, _ = await chunker.chunk(empty_path) + node2, _ = await chunker.chunk(fm_only_path) assert node1.links == [] assert node2.links == [] print("✓ test_parse_links_empty_when_no_content passed") @@ -354,8 +354,8 @@ def test_chunk_does_not_split_wikilink_at_boundary(): temp_path = f.name try: - parser = DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=10) - _, chunks = await parser.parse(temp_path) + chunker = DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=10) + _, chunks = await chunker.chunk(temp_path) # The first chunk must NOT contain a partial link. first = chunks[0].text assert "[[" not in first or "]]" in first, f"first chunk has dangling '[[': {first!r}" @@ -383,8 +383,8 @@ def test_chunk_does_not_split_wikilink_in_overlap(): temp_path = f.name try: - parser = DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=20) - _, chunks = await parser.parse(temp_path) + chunker = DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=20) + _, chunks = await chunker.chunk(temp_path) # No chunk should start mid-link. for c in chunks: t = c.text @@ -413,8 +413,8 @@ def test_chunk_falls_back_for_oversize_link(): temp_path = f.name try: - parser = DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=10) - _, chunks = await parser.parse(temp_path) + chunker = DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=10) + _, chunks = await chunker.chunk(temp_path) # Must terminate (not hang) and cover the whole file. assert len(chunks) >= 2 print("✓ test_chunk_falls_back_for_oversize_link passed") @@ -429,9 +429,9 @@ def test_min_chunk_and_overlap_size(): async def run(): # These values should be clamped to minimums - parser = DefaultFileChunker(chunk_byte_size=1, overlap_byte_size=0) - assert parser.chunk_byte_size == 100 # minimum - assert parser.overlap_byte_size == 4 # minimum + chunker = DefaultFileChunker(chunk_byte_size=1, overlap_byte_size=0) + assert chunker.chunk_byte_size == 100 # minimum + assert chunker.overlap_byte_size == 4 # minimum content = "test" with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f: @@ -439,7 +439,7 @@ def test_min_chunk_and_overlap_size(): temp_path = f.name try: - _, chunks = await parser.parse(temp_path) + _, chunks = await chunker.chunk(temp_path) assert len(chunks) == 1 print("✓ test_min_chunk_and_overlap_size passed") finally: diff --git a/tests4/unit/test_markdown_file_chunker.py b/tests4/unit/test_markdown_file_chunker.py index ad6a87bb..91ebc432 100644 --- a/tests4/unit/test_markdown_file_chunker.py +++ b/tests4/unit/test_markdown_file_chunker.py @@ -46,8 +46,8 @@ def test_parse_empty_file(): async def run(): with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): path = _write_md(tmp, "x.md", "") - parser = MarkdownFileChunker() - node, chunks = await parser.parse(path) + chunker = MarkdownFileChunker() + node, chunks = await chunker.chunk(path) assert node.path == "x.md" assert chunks == [] assert node.links == [] @@ -62,8 +62,8 @@ def test_parse_frontmatter_only(): async def run(): with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): path = _write_md(tmp, "fm.md", "---\nname: t\n---\n") - parser = MarkdownFileChunker() - node, chunks = await parser.parse(path) + chunker = MarkdownFileChunker() + node, chunks = await chunker.chunk(path) assert node.front_matter.name == "t" assert chunks == [] assert node.links == [] @@ -79,8 +79,8 @@ def test_parse_small_body_one_chunk(): with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): body = "# Hello\n\nthis is a small body." path = _write_md(tmp, "small.md", body) - parser = MarkdownFileChunker(chunk_chars=500) - node, chunks = await parser.parse(path) + chunker = MarkdownFileChunker(chunk_chars=500) + node, chunks = await chunker.chunk(path) assert len(chunks) == 1 assert "this is a small body" in chunks[0].text assert node.chunk_ids == [chunks[0].id] @@ -97,8 +97,8 @@ def test_parse_oversized_body_splits(): paras = "\n\n".join(f"paragraph {i} with some content text here." for i in range(50)) body = "# H\n\n" + paras path = _write_md(tmp, "big.md", body) - parser = MarkdownFileChunker(chunk_chars=200) - _, chunks = await parser.parse(path) + chunker = MarkdownFileChunker(chunk_chars=200) + _, chunks = await chunker.chunk(path) assert len(chunks) > 1 print("✓ test_parse_oversized_body_splits passed") @@ -113,8 +113,8 @@ def test_parse_chunk_ids_match_node_chunk_ids(): paras = "\n\n".join(f"para {i} body content here." for i in range(40)) body = "# H\n\n" + paras path = _write_md(tmp, "p.md", body) - parser = MarkdownFileChunker(chunk_chars=200) - node, chunks = await parser.parse(path) + chunker = MarkdownFileChunker(chunk_chars=200) + node, chunks = await chunker.chunk(path) assert node.chunk_ids == [c.id for c in chunks] print("✓ test_parse_chunk_ids_match_node_chunk_ids passed") @@ -128,8 +128,8 @@ def test_parse_links_literal_targets(): with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): body = "see [[topics/Alice.md]] and [[topics/Bob.md#sec]]" path = _write_md(tmp, "note.md", body) - parser = MarkdownFileChunker() - node, _ = await parser.parse(path) + chunker = MarkdownFileChunker() + node, _ = await chunker.chunk(path) triples = {(link.target_path, link.target_anchor, link.predicate) for link in node.links} assert ("topics/Alice.md", None, None) in triples assert ("topics/Bob.md", "sec", None) in triples @@ -154,8 +154,8 @@ def test_parse_links_short_and_no_ext_kept_literally(): with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): body = "see [[Alice]] and [[topics/Alice]] but also [[topics/Alice.md]]" path = _write_md(tmp, "note.md", body) - parser = MarkdownFileChunker() - node, _ = await parser.parse(path) + chunker = MarkdownFileChunker() + node, _ = await chunker.chunk(path) targets = {link.target_path for link in node.links} assert targets == {"Alice", "topics/Alice", "topics/Alice.md"} print("✓ test_parse_links_short_and_no_ext_kept_literally passed") @@ -170,8 +170,8 @@ def test_parse_links_predicate_inline_and_line(): with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): body = "extends:: [[A.md]]\n\nsome [concerns:: [[B.md]]] inline\n" path = _write_md(tmp, "note.md", body) - parser = MarkdownFileChunker() - node, _ = await parser.parse(path) + chunker = MarkdownFileChunker() + node, _ = await chunker.chunk(path) pairs = {(link.target_path, link.predicate) for link in node.links} assert ("A.md", "extends") in pairs assert ("B.md", "concerns") in pairs @@ -187,8 +187,8 @@ def test_parse_links_deduped(): with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): body = "[[A.md]] again [[A.md]] and [[A.md]]" path = _write_md(tmp, "note.md", body) - parser = MarkdownFileChunker() - node, _ = await parser.parse(path) + chunker = MarkdownFileChunker() + node, _ = await chunker.chunk(path) assert len([link for link in node.links if link.target_path == "A.md"]) == 1 print("✓ test_parse_links_deduped passed") @@ -197,8 +197,8 @@ def test_parse_links_deduped(): def test_parse_min_chunk_chars_clamped(): """chunk_chars below 100 should be clamped to 100.""" - parser = MarkdownFileChunker(chunk_chars=10) - assert parser.chunk_chars == 100 + chunker = MarkdownFileChunker(chunk_chars=10) + assert chunker.chunk_chars == 100 print("✓ test_parse_min_chunk_chars_clamped passed") @@ -209,8 +209,8 @@ def test_parse_embed_toc_prefixes_chunk_text(): with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): body = "# Top\n\n## Sub\n\nbody-content" path = _write_md(tmp, "toc.md", body) - parser = MarkdownFileChunker(chunk_chars=200, embed_toc=True) - _, chunks = await parser.parse(path) + chunker = MarkdownFileChunker(chunk_chars=200, embed_toc=True) + _, chunks = await chunker.chunk(path) # Single small section fits; check that the heading appears in text. assert any("Top" in c.text for c in chunks) print("✓ test_parse_embed_toc_prefixes_chunk_text passed")