diff --git a/reme/config/default.yaml b/reme/config/default.yaml index 8f1aa1c8..53ca0f4e 100644 --- a/reme/config/default.yaml +++ b/reme/config/default.yaml @@ -188,7 +188,7 @@ jobs: - messages steps: - backend: auto_memory_step - enable_tags: true + - backend: auto_tag_step auto_memory_cc: backend: base diff --git a/reme/steps/evolve/__init__.py b/reme/steps/evolve/__init__.py index c32b3b9b..fe9e6dbe 100644 --- a/reme/steps/evolve/__init__.py +++ b/reme/steps/evolve/__init__.py @@ -3,6 +3,7 @@ from ._evolve import now, passthrough_response from .auto_image_resource import AutoImageResourceStep from .auto_memory import AutoMemoryStep +from .auto_tag import AutoTagStep from .auto_memory_cc import AutoMemoryCCStep from .auto_resource import AutoResourceStep from .auto_text_resource import AutoTextResourceStep @@ -22,6 +23,7 @@ __all__ = [ "AutoImageResourceStep", "passthrough_response", "AutoMemoryStep", + "AutoTagStep", "AutoMemoryCCStep", "AutoResourceStep", "AutoTextResourceStep", diff --git a/reme/steps/evolve/auto_memory.py b/reme/steps/evolve/auto_memory.py index 8bce1508..a244391d 100644 --- a/reme/steps/evolve/auto_memory.py +++ b/reme/steps/evolve/auto_memory.py @@ -17,8 +17,6 @@ from ...components import R _SESSION_ID_KEY = "session_id" _SOURCE_CONVERSATION_KEY = "source_conversation" -_MAX_TAGS = 8 -_MAX_TAG_LENGTH = 64 _MESSAGE_TIME_ALIASES = ("time_created", "timestamp", "createdAt", "timeCreated", "created_time") @@ -61,39 +59,6 @@ def _normalize_msg_timestamp(item: dict) -> dict: return item -def _normalize_tags(value) -> list[str]: - """Return up to eight unique, retrieval-friendly frontmatter tags. - - Tags are stored uniformly as strings. Technical tokens such as ``GPT-5``, - ``C++``, ``C#``, and ``.NET`` are valid; whitespace-delimited phrases and - punctuation-only values are not. - """ - if not isinstance(value, list): - return [] - - tags: list[str] = [] - seen: set[str] = set() - for item in value: - if isinstance(item, bool) or not isinstance(item, (str, int)): - continue - tag = str(item).strip() - if not tag or len(tag) > _MAX_TAG_LENGTH: - continue - if any(char.isspace() for char in tag): - continue - if not any(char.isalnum() for char in tag): - continue - - dedupe_key = tag.casefold() - if dedupe_key in seen: - continue - seen.add(dedupe_key) - tags.append(tag) - if len(tags) >= _MAX_TAGS: - break - return tags - - @R.register("auto_memory_step") class AutoMemoryStep(BaseStep): """Record conversation facts into a daily note via an Agent.""" @@ -102,7 +67,6 @@ class AutoMemoryStep(BaseStep): super().__init__(**kwargs) self.create_tools: list[str] = ["daily_write"] self.update_tools: list[str] = ["read", "edit", "frontmatter_update", "write"] - self._missing_tag_index_warned = False def _session_dir(self) -> str: return normalize_posix_path(str(self.config_value("session_dir"))) @@ -119,29 +83,6 @@ class AutoMemoryStep(BaseStep): def _daily_note_path(self, day: str, name: str) -> str: return f"{self.config_value('daily_dir')}/{day}/{name}.md" - def _tags_key(self) -> str | None: - """Return the configured tag frontmatter key when tag generation is usable.""" - if not self.kwargs.get("enable_tags", False): - return None - try: - tag_index = self.file_store.tag_index - key = tag_index.tag_key if tag_index is not None else None - except (AttributeError, KeyError, RuntimeError): - key = None - if key: - return key - if not self._missing_tag_index_warned: - self.logger.warning( - f"[{self.name}] enable_tags is true but no tag_index is configured on file_store; " - "tag generation is disabled", - ) - self._missing_tag_index_warned = True - return None - - def _tags_enabled(self) -> bool: - """Whether this step should generate and normalize frontmatter tags.""" - return self._tags_key() is not None - def _frontmatter(self, path: str) -> dict: post = frontmatter.loads((self.file_store.workspace_path / path).read_text(encoding="utf-8")) return dict(post.metadata or {}) @@ -183,9 +124,6 @@ class AutoMemoryStep(BaseStep): _SESSION_ID_KEY: session_id, _SOURCE_CONVERSATION_KEY: self._session_link(session_id), } - tags_key = self._tags_key() - if tags_key is not None: - metadata[tags_key] = _normalize_tags(current.get(tags_key)) if all(current.get(key) == value for key, value in metadata.items()): return response = await self.run_job( @@ -338,6 +276,7 @@ class AutoMemoryStep(BaseStep): # pylint: disable=too-many-return-statements async def execute(self): assert self.context is not None + self.context["modified_paths"] = [] raw_messages = self.context.get("messages") or [] session_id: str = self.context.get("session_id", "") memory_hint: str = self.context.get("memory_hint", "") @@ -399,12 +338,8 @@ class AutoMemoryStep(BaseStep): f"created={created} msgs={len(messages)} hint={bool(memory_hint)}", ) template_key = "user_message_create" if created else "user_message_update" - tags_key = self._tags_key() - enable_tags = tags_key is not None user_message = self.prompt_format( template_key, - enable_tags=enable_tags, - tags_key=tags_key or "tags", today=day, note=memory_hint or "(none)", note_path=note_path, @@ -422,11 +357,7 @@ class AutoMemoryStep(BaseStep): reply_kwargs["injected_job_kwargs"] = {"_allowed_paths": [note_path]} result = await self.agent_wrapper.reply( user_message, - system_prompt=self.prompt_format( - "system_prompt", - enable_tags=enable_tags, - tags_key=tags_key or "tags", - ), + system_prompt=self.prompt_format("system_prompt"), job_tools=self.create_tools if created else self.update_tools, **reply_kwargs, ) @@ -453,11 +384,14 @@ class AutoMemoryStep(BaseStep): return note_path = str(note["path"]) try: - if not created or enable_tags: + if not created: await self._ensure_memory_frontmatter(note_path, session_id) if not created: note_path = await self._rename_from_frontmatter_name(note_path, day) except RuntimeError as exc: + modified = self._note_modified(before_note_path, before_note_bytes, note_path) + if modified and note_path: + self.context["modified_paths"] = [note_path] self.context.response.success = False self.context.response.answer = str(exc) self.context.response.metadata.update( @@ -465,7 +399,7 @@ class AutoMemoryStep(BaseStep): "date": day, "path": note_path, "created": created, - "modified": self._note_modified(before_note_path, before_note_bytes, note_path), + "modified": modified, "n_messages": len(messages), }, ) @@ -473,6 +407,8 @@ class AutoMemoryStep(BaseStep): return modified = self._note_modified(before_note_path, before_note_bytes, note_path) + if modified: + self.context["modified_paths"] = [note_path] daily_dir = self.config_value("daily_dir") self.logger.info(f"[{self.name}] refresh index start date={day} daily_dir={daily_dir}") index_payload = await refresh_day_index(self.file_store, day, daily_dir) diff --git a/reme/steps/evolve/auto_memory.yaml b/reme/steps/evolve/auto_memory.yaml index fa0d7619..0767a502 100644 --- a/reme/steps/evolve/auto_memory.yaml +++ b/reme/steps/evolve/auto_memory.yaml @@ -20,7 +20,6 @@ system_prompt: | - `name` = a concise, stable topic/event filename stem, such as `cold-remedies` or `project-kickoff-decision`. Do not include today's date or the daily directory date; the outer daily path already records the date. For existing notes, update it when a better filename is clearly warranted. - `description` = a thorough summary; vague descriptions like "notes" / "misc" are unacceptable. - [enable_tags] - `{tags_key}` = 0–8 unique retrieval keywords for the complete note. Always include the field, using `[]` when no useful tags exist. Each tag must be one string with no whitespace, may contain technical punctuation (for example `GPT-5`, `C++`, `C#`, or `.NET`), must contain at least one letter or digit, and must be at most 64 characters. Store numeric tags as strings, for example `"100"`. - **Never set `status`** — it is a field reserved for downstream processing. system_prompt_zh: | 你是自动记忆系统。你的职责是将最近对话中的核心信息记录到日记记忆中。思考人类会从这段对话中自然地记住什么——不是所有内容，而是真正重要的信息。 @@ -44,7 +43,6 @@ system_prompt_zh: | - `name` = 简洁、稳定的主题/事件文件名 stem，例如 `cold-remedies` 或 `project-kickoff-decision`。不要包含今天日期或日记目录日期；外层日记路径已经记录日期。对已有笔记，如果明显有更好的文件名，就更新它。 - `description` = 详细总结；模糊的描述如 "notes" / "misc" 不可接受。 - [enable_tags] - `{tags_key}` = 针对完整笔记的 0–8 个不重复检索关键词。该字段必须始终存在；没有合适关键词时使用 `[]`。每个 tag 必须是一个不含空白字符的字符串，可以包含技术名称中的标点（例如 `GPT-5`、`C++`、`C#` 或 `.NET`），必须至少包含一个字母或数字，且不得超过 64 个字符。数字 tag 也保存为字符串，例如 `"100"`。 - **永远不要设置 `status`**——它是下游处理保留的字段。 user_message_create: | @@ -70,13 +68,11 @@ user_message_create: | Create the note in one shot: `daily_write name= description= session_id={session_id} date={today} content=` - [enable_tags] Also pass `metadata={{"{tags_key}": [, ...]}}` in the same `daily_write` call. - Generate `name` as a concise, stable topic/event filename stem for this memory. Prefer a reusable topic or event summary, optionally in kebab-case. - Do not include today's date or the daily directory date in `name`; the note already lives under today's daily path. - `name` must be a valid single filename component: no slash, backslash, leading/trailing whitespace, or characters like `< > : " | ? *`. - `description` must be a thorough summary of the body — specific enough that the description alone conveys all key information. - [enable_tags] - Generate 0–8 `{tags_key}` values from the complete note. Always pass `metadata={{"{tags_key}": [...]}}`, including an empty list when there are no useful tags. Follow the system prompt's tag format exactly. ## Step 3 — Summary @@ -108,13 +104,11 @@ user_message_create_zh: | 一次性创建笔记： `daily_write name= description= session_id={session_id} date={today} content=<正文>` - [enable_tags] 在同一次 `daily_write` 调用中另外传入 `metadata={{"{tags_key}": [, ...]}}`。 - 由你生成 `name`，作为这条记忆简洁、稳定的主题/事件文件名 stem。优先使用可复用的主题或事件总结，可以采用 kebab-case。 - `name` 不要包含今天日期或日记目录日期；笔记已经位于当天日记路径下。 - `name` 必须是合法的单个文件名组件：不能包含 slash、反斜杠、首尾空白，或 `< > : " | ? *` 等字符。 - `description` 必须是正文的详尽总结——具体到仅凭 description 就能传达全部核心信息。 - [enable_tags] - 根据完整笔记生成 0–8 个 `{tags_key}` 值。必须始终传入 `metadata={{"{tags_key}": [...]}}`，没有合适关键词时也要传入空数组。严格遵守 system prompt 的 tag 格式。 ## 步骤 3 — 总结 @@ -161,22 +155,18 @@ user_message_update: | Execution: 1. Use `edit path={note_path} old= new=` for each section that needs updating. You may call `edit` multiple times. 2. After body changes, refresh frontmatter with `frontmatter_update path={note_path} metadata={{"name": "", "description": ""}}`. - [enable_tags] Also regenerate 0–8 tags from the complete merged note and include `"{tags_key}": [, ...]` in the same metadata object. Always include `{tags_key}`, using `[]` when none are useful. - Keep the existing `name` only when it is already the best concise topic/event filename stem. The system will rename the file after your final response. - Do not add today's date or the daily directory date to `name`. 3. If `edit` fails repeatedly (e.g., cannot find the original text due to formatting mismatch), fall back to `write path={note_path} name= description= content=` for a complete rewrite. - [enable_tags] In this fallback, also pass `metadata={{"{tags_key}": [, ...]}}`. ## Step 3b — Full Write (Empty File Fallback) The file exists but its body is empty. Write the full content in one shot: `write path={note_path} name= description= content=` - [enable_tags] Also pass `metadata={{"{tags_key}": [, ...]}}` in this call. - Use a concise, stable topic/event `name`; filename changes are applied after your final response. - Do not include today's date or the daily directory date in `name`. - `description` must be a thorough summary of the body — specific enough that the description alone conveys all key information. - [enable_tags] - Generate 0–8 tags from the complete body and always include `metadata={{"{tags_key}": [...]}}`, using `[]` when none are useful. ## Step 4 — Summary @@ -224,22 +214,18 @@ user_message_update_zh: | 执行： 1. 对需要更新的每个部分使用 `edit path={note_path} old=<原文片段> new=<替换片段>`。可以多次调用 `edit`。 2. 正文变更后，刷新 frontmatter：`frontmatter_update path={note_path} metadata={{"name": "<更新后的文件名 stem>", "description": "<更新后的总结>"}}`。 - [enable_tags] 同时根据合并后的完整笔记重新生成 0–8 个 tags，并在同一个 metadata 对象中包含 `"{tags_key}": [, ...]`。必须始终包含 `{tags_key}`，没有合适关键词时使用 `[]`。 - 只有当前 `name` 已经是最合适的简洁主题/事件文件名 stem 时才保留。系统会在你最终回复后负责重命名文件。 - 不要在 `name` 中加入今天日期或日记目录日期。 3. 如果 `edit` 多次失败（如因格式不匹配找不到原文），退回 `write path={note_path} name= description= content=<完整正文>` 全量重写。 - [enable_tags] 在该 fallback 中还要传入 `metadata={{"{tags_key}": [, ...]}}`。 ## 步骤 3b — 全量写入（空文件 fallback） 文件存在但正文为空。一次性写入完整内容： `write path={note_path} name= description= content=<正文>` - [enable_tags] 在该调用中另外传入 `metadata={{"{tags_key}": [, ...]}}`。 - 使用简洁、稳定的主题/事件 `name`；文件名变化会在你最终回复后应用。 - `name` 不要包含今天日期或日记目录日期。 - `description` 必须是正文的详尽总结——具体到仅凭 description 就能传达全部核心信息。 - [enable_tags] - 根据完整正文生成 0–8 个 tags，并始终包含 `metadata={{"{tags_key}": [...]}}`；没有合适关键词时使用 `[]`。 ## 步骤 4 — 总结 diff --git a/reme/steps/evolve/auto_tag.py b/reme/steps/evolve/auto_tag.py new file mode 100644 index 00000000..d05c1e64 --- /dev/null +++ b/reme/steps/evolve/auto_tag.py @@ -0,0 +1,148 @@ +"""auto_tag — generate frontmatter tags for modified Markdown files.""" + +from pathlib import Path + +import frontmatter + +from ._evolve import agent_reply_result_text +from ..base_step import BaseStep +from ..file_io import parse_daily_date, refresh_day_index +from ..file_io._path import display_path, resolve_path +from ..index import normalize_posix_path +from ...components import R + +_MAX_TAGS = 8 +_MAX_TAG_LENGTH = 64 + + +def normalize_tags(value) -> list[str]: + """Return unique retrieval-friendly tags that satisfy the storage contract.""" + if not isinstance(value, list): + return [] + + tags: list[str] = [] + seen: set[str] = set() + for item in value: + if isinstance(item, bool) or not isinstance(item, (str, int)): + continue + tag = str(item).strip() + if not tag or len(tag) > _MAX_TAG_LENGTH: + continue + if any(char.isspace() for char in tag): + continue + if not any(char.isalnum() for char in tag): + continue + dedupe_key = tag.casefold() + if dedupe_key in seen: + continue + seen.add(dedupe_key) + tags.append(tag) + if len(tags) >= _MAX_TAGS: + break + return tags + + +@R.register("auto_tag_step") +class AutoTagStep(BaseStep): + """Generate tags for each eligible path produced by an earlier Step.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.tools = ["read", "list_tags", "frontmatter_read", "frontmatter_update"] + + def _eligible_paths(self) -> tuple[list[str], list[str]]: + assert self.context is not None + raw_paths = self.context.get("modified_paths") or [] + if not isinstance(raw_paths, list): + raw_paths = [raw_paths] + + workspace = Path(self.file_store.workspace_path or ".").resolve() + eligible: list[str] = [] + ignored: list[str] = [] + seen: set[str] = set() + for raw_path in raw_paths: + raw = str(raw_path or "").strip() + target, err = resolve_path(workspace, raw) + if err or target is None or not target.is_file() or target.suffix.lower() != ".md": + ignored.append(raw) + continue + path = normalize_posix_path(display_path(workspace, target)) + if path in seen: + continue + seen.add(path) + eligible.append(path) + return eligible, ignored + + async def _process_path(self, path: str, tag_key: str) -> str: + result = await self.agent_wrapper.reply( + self.prompt_format("user_message", path=path, tags_key=tag_key), + system_prompt=self.prompt_format("system_prompt", tags_key=tag_key), + job_tools=self.tools, + injected_job_kwargs={"_allowed_paths": [path]}, + ) + + target = Path(self.file_store.workspace_path or ".") / path + metadata = dict(frontmatter.loads(target.read_text(encoding="utf-8")).metadata or {}) + normalized = normalize_tags(metadata.get(tag_key)) + if metadata.get(tag_key) != normalized: + response = await self.run_job( + "frontmatter_update", + path=path, + metadata={tag_key: normalized}, + _allowed_paths=[path], + ) + if not response.success: + raise RuntimeError(str(response.answer)) + return agent_reply_result_text(result) + + def _daily_date(self, path: str) -> str | None: + daily_dir = normalize_posix_path(str(self.config_value("daily_dir"))).strip("/") + prefix = f"{daily_dir}/" + if not path.startswith(prefix): + return None + remainder = path[len(prefix) :] + parts = remainder.split("/") + if len(parts) != 2: + return None + return parse_daily_date(parts[0]) + + async def execute(self): + assert self.context is not None + initial_success = self.context.response.success + tag_index = getattr(self.file_store, "tag_index", None) + if tag_index is None: + self.context.response.success = False + self.context.response.answer = "Error: tag index is not configured" + return self.context.response + + eligible, ignored = self._eligible_paths() + tagged: list[str] = [] + failed: list[dict[str, str]] = [] + summaries: list[str] = [] + dates: set[str] = set() + for path in eligible: + if day := self._daily_date(path): + dates.add(day) + try: + summaries.append(await self._process_path(path, tag_index.tag_key)) + tagged.append(path) + except Exception as exc: # pylint: disable=broad-exception-caught + failed.append({"path": path, "error": str(exc)}) + self.logger.warning(f"[{self.name}] failed path={path}: {exc}") + + indexes = [ + await refresh_day_index(self.file_store, day, self.config_value("daily_dir")) for day in sorted(dates) + ] + self.context.response.success = initial_success and not failed + if failed: + self.context.response.answer = f"Tagged {len(tagged)} file(s); {len(failed)} failed" + elif tagged: + self.context.response.answer = f"Tagged {len(tagged)} file(s)" + self.context.response.metadata["auto_tag"] = { + "tagged_paths": tagged, + "ignored_paths": ignored, + "failed_paths": failed, + "indexes": indexes, + "summaries": [summary for summary in summaries if summary], + } + return self.context.response diff --git a/reme/steps/evolve/auto_tag.yaml b/reme/steps/evolve/auto_tag.yaml new file mode 100644 index 00000000..eecfcd7a --- /dev/null +++ b/reme/steps/evolve/auto_tag.yaml @@ -0,0 +1,29 @@ +system_prompt: | + You add retrieval tags to one Markdown memory file at a time. + + Read the complete target file and its frontmatter, then inspect existing tags with `list_tags`. Update only the + `{tags_key}` frontmatter field by calling `frontmatter_update`. Do not change the body or any other frontmatter field. + + Generate 0–8 unique retrieval keywords for the complete note. Always write `{tags_key}`, using `[]` when no useful + tags exist. Each tag must be one string with no whitespace, may contain technical punctuation such as `GPT-5`, `C++`, + `C#`, or `.NET`, must contain at least one letter or digit, and must be at most 64 characters. Store numeric tags as + strings. Prefer an existing tag when it accurately expresses the same concept. +system_prompt_zh: | + 你每次为一个 Markdown 记忆文件添加检索 tag。 + + 阅读目标文件的完整内容和 frontmatter，然后通过 `list_tags` 查看已有 tag。只允许调用 `frontmatter_update` + 更新 frontmatter 的 `{tags_key}` 字段，不得修改正文或其他 frontmatter 字段。 + + 根据完整笔记生成 0–8 个不重复检索关键词。必须始终写入 `{tags_key}`，没有合适关键词时使用 `[]`。每个 tag + 必须是一个不含空白字符的字符串，可以包含 `GPT-5`、`C++`、`C#`、`.NET` 等技术标点，必须至少包含一个 + 字母或数字，且不得超过 64 个字符。数字 tag 也保存为字符串。含义相同时优先复用已有 tag。 +user_message: | + Target path: {path} + Tag field: {tags_key} + + Read this file, select tags for its complete content, and update only its `{tags_key}` frontmatter field. +user_message_zh: | + 目标路径：{path} + Tag 字段：{tags_key} + + 阅读该文件，根据完整内容选择 tag，并且只更新 frontmatter 的 `{tags_key}` 字段。 diff --git a/tests/unit/test_auto_tag.py b/tests/unit/test_auto_tag.py new file mode 100644 index 00000000..0c689433 --- /dev/null +++ b/tests/unit/test_auto_tag.py @@ -0,0 +1,165 @@ +"""Focused tests for the standalone automatic tagging Step.""" + +# pylint: disable=missing-function-docstring + +from pathlib import Path + +import frontmatter +import pytest + +from reme.components.agent_wrapper import BaseAgentWrapper +from reme.components.file_store import LocalFileStore +from reme.components.runtime_context import RuntimeContext +from reme.components.tag_index import LocalTagIndex +from reme.schema import Response +from reme.steps.evolve.auto_tag import AutoTagStep, normalize_tags + + +class _TaggingWrapper(BaseAgentWrapper): + def __init__( + self, + workspace: Path, + *, + fail_name: str = "", + tag_key: str = "tags", + tags: list[object] | None = None, + ) -> None: + super().__init__(name="tagger") + self.workspace = workspace + self.fail_name = fail_name + self.tag_key = tag_key + self.tags = ["ReMe", "Python"] if tags is None else tags + self.calls: list[tuple[str, dict]] = [] + + async def reply(self, inputs, **kwargs) -> dict: + text = str(inputs) + self.calls.append((text, kwargs)) + path = kwargs["injected_job_kwargs"]["_allowed_paths"][0] + if Path(path).name == self.fail_name: + raise RuntimeError("tagging failed") + target = self.workspace / path + post = frontmatter.loads(target.read_text(encoding="utf-8")) + post.metadata[self.tag_key] = self.tags + target.write_text(frontmatter.dumps(post), encoding="utf-8") + return {"result": f"tagged {path}", "last_message": {}} + + +def _write_note(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("---\nname: note\ndescription: useful note\n---\nbody\n", encoding="utf-8") + + +@pytest.mark.asyncio +async def test_auto_tag_requires_tag_index_before_modifying_files(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + note = tmp_path / "daily/2026-09-09/note.md" + _write_note(note) + before = note.read_bytes() + store = LocalFileStore(name="store", embedding_store="", tag_index="") + wrapper = _TaggingWrapper(tmp_path) + step = AutoTagStep(file_store=store, agent_wrapper=wrapper) + + response = await step(RuntimeContext(modified_paths=["daily/2026-09-09/note.md"])) + + assert response.success is False + assert response.answer == "Error: tag index is not configured" + assert not wrapper.calls + assert note.read_bytes() == before + + +@pytest.mark.asyncio +async def test_auto_tag_filters_paths_and_continues_after_one_file_fails(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + first = tmp_path / "daily/2026-09-09/first.md" + failed = tmp_path / "daily/2026-09-09/failed.md" + _write_note(first) + _write_note(failed) + (tmp_path / "daily/2026-09-09/notes").mkdir() + (tmp_path / "daily/2026-09-09/plain.txt").write_text("text", encoding="utf-8") + + store = LocalFileStore(name="store", embedding_store="", tag_index="") + store.tag_index = LocalTagIndex() + wrapper = _TaggingWrapper(tmp_path, fail_name="failed.md") + step = AutoTagStep(file_store=store, agent_wrapper=wrapper) + context = RuntimeContext( + modified_paths=[ + "daily/2026-09-09/failed.md", + "daily/2026-09-09/notes", + "daily/2026-09-09/plain.txt", + "daily/2026-09-09/first.md", + "daily/2026-09-09/first.md", + ], + ) + + response = await step(context) + + assert response.success is False + assert [call[1]["injected_job_kwargs"] for call in wrapper.calls] == [ + {"_allowed_paths": ["daily/2026-09-09/failed.md"]}, + {"_allowed_paths": ["daily/2026-09-09/first.md"]}, + ] + assert all( + call[1]["job_tools"] == ["read", "list_tags", "frontmatter_read", "frontmatter_update"] + for call in wrapper.calls + ) + assert frontmatter.loads(first.read_text(encoding="utf-8")).metadata["tags"] == ["ReMe", "Python"] + assert "tags" not in frontmatter.loads(failed.read_text(encoding="utf-8")).metadata + assert response.metadata["auto_tag"]["tagged_paths"] == ["daily/2026-09-09/first.md"] + assert response.metadata["auto_tag"]["ignored_paths"] == [ + "daily/2026-09-09/notes", + "daily/2026-09-09/plain.txt", + ] + assert response.metadata["auto_tag"]["failed_paths"] == [ + {"path": "daily/2026-09-09/failed.md", "error": "tagging failed"}, + ] + assert "tags: ['ReMe', 'Python']" in (tmp_path / "daily/2026-09-09.md").read_text(encoding="utf-8") + + +@pytest.mark.asyncio +async def test_auto_tag_uses_configured_key_and_normalizes_agent_output(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + note = tmp_path / "memory/note.md" + _write_note(note) + store = LocalFileStore(name="store", embedding_store="", tag_index="") + store.tag_index = LocalTagIndex(tag_key="keywords") + wrapper = _TaggingWrapper( + tmp_path, + tag_key="keywords", + tags=["ReMe", "reme", "memory system", "++", 100], + ) + step = AutoTagStep(file_store=store, agent_wrapper=wrapper) + + async def update_frontmatter(name, /, **kwargs): + assert name == "frontmatter_update" + post = frontmatter.loads(note.read_text(encoding="utf-8")) + post.metadata.update(kwargs["metadata"]) + note.write_text(frontmatter.dumps(post), encoding="utf-8") + return Response(answer="updated") + + monkeypatch.setattr(step, "run_job", update_frontmatter) + + response = await step(RuntimeContext(modified_paths=["memory/note.md"])) + + assert response.success is True + assert frontmatter.loads(note.read_text(encoding="utf-8")).metadata["keywords"] == ["ReMe", "100"] + assert "Tag field: keywords" in wrapper.calls[0][0] + assert "`keywords`" in wrapper.calls[0][1]["system_prompt"] + + +def test_normalize_tags_enforces_storage_contract(): + assert normalize_tags( + [ + "GPT-5", + "C++", + "C#", + ".NET", + 100, + "memory system", + "++", + "ReMe", + "reme", + "tag7", + "tag8", + "tag9", + ], + ) == ["GPT-5", "C++", "C#", ".NET", "100", "ReMe", "tag7", "tag8"] diff --git a/tests/unit/test_background_steps.py b/tests/unit/test_background_steps.py index 451f4021..5cc5c2d1 100644 --- a/tests/unit/test_background_steps.py +++ b/tests/unit/test_background_steps.py @@ -2012,28 +2012,29 @@ def test_auto_memory_reports_modified_for_create_and_false_for_skip(): app_context=app_ctx, file_store=fs, agent_wrapper=wrapper, - enable_tags=True, ) - resp = await step( - RuntimeContext( - messages=[{"name": "user", "role": "user", "content": "remember project detail"}], - session_id="s1", - ), + context = RuntimeContext( + messages=[{"name": "user", "role": "user", "content": "remember project detail"}], + session_id="s1", ) + resp = await step(context) resp = resp or step.context.response assert resp.success is True assert resp.metadata["created"] is True assert resp.metadata["modified"] is True - assert "tags: []" in (cwd / "daily" / today / "memory.md").read_text(encoding="utf-8") + assert context["modified_paths"] == [f"daily/{today}/memory.md"] + assert "tags:" not in (cwd / "daily" / today / "memory.md").read_text(encoding="utf-8") wrapper.on_reply = None - resp = await step(RuntimeContext(messages=[], session_id="s2")) + context = RuntimeContext(messages=[], session_id="s2") + resp = await step(context) resp = resp or step.context.response assert resp.success is True assert resp.metadata["modified"] is False assert resp.metadata["n_messages"] == 0 + assert context["modified_paths"] == [] finally: await fs.close() print("✓ test_auto_memory_reports_modified_for_create_and_false_for_skip passed") @@ -2041,104 +2042,6 @@ def test_auto_memory_reports_modified_for_create_and_false_for_skip(): asyncio.run(run()) -def test_auto_memory_normalizes_tags_after_existing_note_update(): - """Existing notes receive a refreshed, normalized tags field capped at eight entries.""" - - async def run(): - with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): - cwd = Path.cwd() - app_ctx = _make_app_context(cwd) - fs = LocalFileStore(name="test_store", embedding_store="", tag_index="default") - wrapper = _FakeAgentWrapper() - await fs.start() - _install_file_jobs(app_ctx, fs) - try: - today = datetime.datetime.now().strftime("%Y-%m-%d") - note_path = cwd / "daily" / today / "memory.md" - write_file( - note_path, - "---\nname: memory\nsession_id: s1\n" - "source_conversation: '[[session/dialog/s1.jsonl]]'\n" - "tags: [old]\n---\nold body\n", - ) - - wrapper.on_reply = lambda *_: write_file( - note_path, - "---\nname: memory\nsession_id: s1\n" - "source_conversation: '[[session/dialog/s1.jsonl]]'\n" - "tags: [GPT-5, C++, C#, .NET, 100, 'memory system', '++', ReMe, reme, tag7, tag8, tag9]\n" - "---\nupdated body\n", - ) - - step = AutoMemoryStep( - app_context=app_ctx, - file_store=fs, - agent_wrapper=wrapper, - enable_tags=True, - ) - resp = await step( - RuntimeContext( - messages=[{"name": "user", "role": "user", "content": "updated project detail"}], - session_id="s1", - ), - ) - resp = resp or step.context.response - - assert resp.success is True - metadata = step._frontmatter(f"daily/{today}/memory.md") - assert metadata["tags"] == ["GPT-5", "C++", "C#", ".NET", "100", "ReMe", "tag7", "tag8"] - finally: - await fs.close() - - asyncio.run(run()) - - -def test_auto_memory_uses_tag_index_frontmatter_key(): - """Generate and normalize metadata under the key selected by the bound tag index.""" - - async def run(): - with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): - cwd = Path.cwd() - app_ctx = _make_app_context(cwd) - fs = LocalFileStore(name="test_store", embedding_store="", tag_index="default") - wrapper = _FakeAgentWrapper() - await fs.start() - fs.tag_index.tag_key = "keywords" - _install_file_jobs(app_ctx, fs) - try: - today = datetime.datetime.now().strftime("%Y-%m-%d") - note_path = cwd / "daily" / today / "memory.md" - wrapper.on_reply = lambda *_: write_file( - note_path, - "---\nname: memory\nsession_id: s1\n" - "source_conversation: '[[session/dialog/s1.jsonl]]'\n" - "keywords: [ReMe]\n---\nbody\n", - ) - - step = AutoMemoryStep( - app_context=app_ctx, - file_store=fs, - agent_wrapper=wrapper, - enable_tags=True, - ) - resp = await step( - RuntimeContext( - messages=[{"name": "user", "role": "user", "content": "remember ReMe"}], - session_id="s1", - ), - ) - resp = resp or step.context.response - - assert resp.success is True - assert step._frontmatter(f"daily/{today}/memory.md")["keywords"] == ["ReMe"] - assert 'metadata={"keywords"' in wrapper.inputs - assert "`keywords`" in wrapper.kwargs["system_prompt"] - finally: - await fs.close() - - asyncio.run(run()) - - def test_auto_memory_uses_message_day_for_historical_create(): """AutoMemoryStep creates historical daily notes from message timestamps.""" diff --git a/tests/unit/test_evolve_utils.py b/tests/unit/test_evolve_utils.py index b8e3312c..8c890b23 100644 --- a/tests/unit/test_evolve_utils.py +++ b/tests/unit/test_evolve_utils.py @@ -9,7 +9,8 @@ from agentscope.message import Msg import pytest from reme.steps.evolve._evolve import agent_reply_result_text, format_history -from reme.steps.evolve.auto_memory import AutoMemoryStep, _normalize_tags, _sanitize_msg_for_save +from reme.steps.evolve.auto_memory import AutoMemoryStep, _sanitize_msg_for_save +from reme.steps.evolve.auto_tag import normalize_tags def test_agent_reply_result_text_uses_last_text_block(): @@ -75,9 +76,9 @@ def test_sanitize_msg_for_save_drops_tool_results_and_base64_data(): assert sanitized.content[1].name == "memory_search" -def test_auto_memory_normalizes_frontmatter_tags(): +def test_auto_tag_normalizes_frontmatter_tags(): """Tags keep technical punctuation, reject phrases, de-duplicate, and stop at eight.""" - assert _normalize_tags( + assert normalize_tags( [ "GPT-5", "C++", @@ -95,10 +96,10 @@ def test_auto_memory_normalizes_frontmatter_tags(): ], ) == ["GPT-5", "C++", "C#", ".NET", "100", "ReMe", "tag7", "tag8"] # pylint: disable=use-implicit-booleaness-not-comparison - assert _normalize_tags(None) == [] - assert _normalize_tags("GPT-5") == [] + assert normalize_tags(None) == [] + assert normalize_tags("GPT-5") == [] # pylint: enable=use-implicit-booleaness-not-comparison - assert _normalize_tags(["x" * 65, True, {}, "valid"]) == ["valid"] + assert normalize_tags(["x" * 65, True, {}, "valid"]) == ["valid"] def test_auto_memory_accepts_message_timestamp_aliases(): diff --git a/tests/unit/test_injected_job_kwargs.py b/tests/unit/test_injected_job_kwargs.py index 07058817..a2c7cd02 100644 --- a/tests/unit/test_injected_job_kwargs.py +++ b/tests/unit/test_injected_job_kwargs.py @@ -8,16 +8,12 @@ the injected keys from the tool schema exposed to the model. # pylint: disable=protected-access,missing-function-docstring -from unittest.mock import MagicMock - import pytest from reme.components.agent_wrapper import AsAgentWrapper, BaseAgentWrapper, CcAgentWrapper from reme.components.file_store import LocalFileStore -from reme.components.tag_index import LocalTagIndex from reme.schema import Response from reme.steps.evolve.auto_memory import AutoMemoryStep -from reme.steps.evolve.auto_memory_cc import AutoMemoryCCStep class _Job: @@ -240,25 +236,7 @@ def test_auto_memory_keeps_original_tool_names(): assert step.update_tools == ["read", "edit", "frontmatter_update", "write"] -def test_auto_memory_tags_are_disabled_by_default_and_enabled_through_kwargs(): - assert AutoMemoryStep()._tags_enabled() is False - assert AutoMemoryCCStep()._tags_enabled() is False - store = LocalFileStore(embedding_store="", tag_index="") - store.tag_index = LocalTagIndex(tag_key="keywords") - assert AutoMemoryStep(enable_tags=True, file_store=store)._tags_enabled() is True - assert AutoMemoryStep(enable_tags=True)._tags_enabled() is False - - -def test_auto_memory_warns_once_when_tags_enabled_without_tag_index(): - step = AutoMemoryStep(enable_tags=True) - step.logger = MagicMock() - - assert step._tags_enabled() is False - assert step._tags_enabled() is False - step.logger.warning.assert_called_once() - - -def test_auto_memory_tags_prompt_follows_step_kwarg(): +def test_auto_memory_prompt_has_no_tag_responsibility(): step = AutoMemoryStep() prompt_kwargs = { "today": "2026-09-01", @@ -267,42 +245,16 @@ def test_auto_memory_tags_prompt_follows_step_kwarg(): "history": "user: remember GPT-5", } - disabled_system = step.prompt_format("system_prompt", enable_tags=step._tags_enabled()) - disabled_create = step.prompt_format("user_message_create", enable_tags=step._tags_enabled(), **prompt_kwargs) - disabled_update = step.prompt_format( + system = step.prompt_format("system_prompt") + create = step.prompt_format("user_message_create", **prompt_kwargs) + update = step.prompt_format( "user_message_update", - enable_tags=step._tags_enabled(), note_path="daily/2026-09-01/memory.md", **prompt_kwargs, ) - assert "`tags`" not in disabled_system - assert 'metadata={"tags"' not in disabled_create - assert '"tags"' not in disabled_update - - store = LocalFileStore(embedding_store="", tag_index="") - store.tag_index = LocalTagIndex(tag_key="keywords") - enabled_step = AutoMemoryStep(enable_tags=True, file_store=store) - enabled_system = enabled_step.prompt_format( - "system_prompt", - enable_tags=enabled_step._tags_enabled(), - tags_key=enabled_step._tags_key(), - ) - enabled_create = enabled_step.prompt_format( - "user_message_create", - enable_tags=enabled_step._tags_enabled(), - tags_key=enabled_step._tags_key(), - **prompt_kwargs, - ) - enabled_update = enabled_step.prompt_format( - "user_message_update", - enable_tags=enabled_step._tags_enabled(), - tags_key=enabled_step._tags_key(), - note_path="daily/2026-09-01/memory.md", - **prompt_kwargs, - ) - assert "`keywords`" in enabled_system - assert 'metadata={"keywords"' in enabled_create - assert '"keywords"' in enabled_update + assert "tag" not in system.casefold() + assert "tag" not in create.casefold() + assert "tag" not in update.casefold() def test_auto_memory_create_prompts_match_upstream_date_arguments(): @@ -312,8 +264,8 @@ def test_auto_memory_create_prompts_match_upstream_date_arguments(): prompt_file = Path("reme/steps/evolve/auto_memory.yaml") evolve_prompt = prompt_file.read_text(encoding="utf-8") assert "date={today}" in evolve_prompt or "`date`: {today}" in evolve_prompt or "`date`：{today}" in evolve_prompt - assert '"{tags_key}": [, ...]' in evolve_prompt - assert '"{tags_key}": []' in evolve_prompt or "using `[]`" in evolve_prompt + assert "enable_tags" not in evolve_prompt + assert "tags_key" not in evolve_prompt def test_configs_define_original_jobs_without_daily_variants(): @@ -328,7 +280,10 @@ def test_configs_define_original_jobs_without_daily_variants(): assert name not in jobs, f"{config_name} unexpectedly defines {name}" default = resolve_app_config(config="default", log_config=False) - assert default["jobs"]["auto_memory"]["steps"][0]["enable_tags"] is True + assert default["jobs"]["auto_memory"]["steps"] == [ + {"backend": "auto_memory_step"}, + {"backend": "auto_tag_step"}, + ] if __name__ == "__main__": diff --git a/tests/unit/test_tag_index.py b/tests/unit/test_tag_index.py index 552b8cd3..64bd1513 100644 --- a/tests/unit/test_tag_index.py +++ b/tests/unit/test_tag_index.py @@ -539,4 +539,7 @@ def test_default_config_enables_tag_index_with_explicit_key() -> None: assert config["components"]["tag_index"]["default"]["tag_key"] == "tags" assert config["components"]["file_store"]["default"]["tag_index"] == "default" assert config["jobs"]["search"]["parameters"]["properties"]["tags"]["default"] == [] - assert config["jobs"]["auto_memory"]["steps"][0]["enable_tags"] is True + assert config["jobs"]["auto_memory"]["steps"] == [ + {"backend": "auto_memory_step"}, + {"backend": "auto_tag_step"}, + ]