From b3a0ba62b2aac5de5df3d0270ca0487bf7d268fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=85=B1=E7=89=9B=E8=82=89?= Date: Thu, 10 Sep 2026 16:37:50 +0800 Subject: [PATCH] Make max_tags_per_file configurable in auto-tag step --- reme/config/default.yaml | 1 + reme/steps/evolve/auto_tag.py | 66 +++++++++++++++++++++----- reme/steps/evolve/auto_tag.yaml | 12 ++--- tests/unit/test_auto_tag.py | 29 +++++++++-- tests/unit/test_injected_job_kwargs.py | 2 +- tests/unit/test_tag_index.py | 2 +- 6 files changed, 90 insertions(+), 22 deletions(-) diff --git a/reme/config/default.yaml b/reme/config/default.yaml index 561d3962..2df9707d 100644 --- a/reme/config/default.yaml +++ b/reme/config/default.yaml @@ -189,6 +189,7 @@ jobs: steps: - backend: auto_memory_step - backend: auto_tag_step + max_tags_per_file: 3 auto_memory_cc: backend: base diff --git a/reme/steps/evolve/auto_tag.py b/reme/steps/evolve/auto_tag.py index 3324507e..1abd4a33 100644 --- a/reme/steps/evolve/auto_tag.py +++ b/reme/steps/evolve/auto_tag.py @@ -13,8 +13,8 @@ from ..file_io._path import display_path, resolve_path from ..index import normalize_posix_path from ...components import R -_MAX_MEMORY_TAGS = 3 -_MAX_MEMORY_TAG_LENGTH = 64 +_DEFAULT_MAX_MEMORY_TAGS = 3 +_DEFAULT_MAX_MEMORY_TAG_LENGTH = 64 _SUPPORTED_CHANGES = {"added", "modified"} @@ -24,8 +24,21 @@ class _TagTarget: path: str -def normalize_memory_tags(value: object) -> list[str]: - """Normalize up to three human-readable entity labels for frontmatter storage.""" +def _positive_int(value: object, *, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + return value + + +def normalize_memory_tags( + value: object, + *, + max_tags_per_file: int = _DEFAULT_MAX_MEMORY_TAGS, + max_tag_length: int = _DEFAULT_MAX_MEMORY_TAG_LENGTH, +) -> list[str]: + """Normalize human-readable entity labels for frontmatter storage.""" + max_tags_per_file = _positive_int(max_tags_per_file, name="max_tags_per_file") + max_tag_length = _positive_int(max_tag_length, name="max_tag_length") if not isinstance(value, list): return [] @@ -35,14 +48,14 @@ def normalize_memory_tags(value: object) -> list[str]: if not isinstance(item, str): continue tag = " ".join(item.split()) - if not tag or len(tag) > _MAX_MEMORY_TAG_LENGTH or not any(char.isalnum() for char in tag): + if not tag or len(tag) > max_tag_length or not any(char.isalnum() for char in tag): continue canonical = tag.casefold() if canonical in seen: continue seen.add(canonical) tags.append(tag) - if len(tags) >= _MAX_MEMORY_TAGS: + if len(tags) >= max_tags_per_file: break return tags @@ -51,10 +64,22 @@ def normalize_memory_tags(value: object) -> list[str]: class AutoTagStep(BaseStep): """Update memory tags for Markdown files described by the common ``changes`` contract.""" - def __init__(self, **kwargs): + def __init__(self, max_tags_per_file: int = _DEFAULT_MAX_MEMORY_TAGS, **kwargs): super().__init__(**kwargs) + self.max_tags_per_file = _positive_int(max_tags_per_file, name="max_tags_per_file") self.tools = ["read", "list_tags", "frontmatter_read", "frontmatter_update"] + @staticmethod + def _index_limit(tag_index, name: str, fallback: int | None) -> int | None: + """Read one positive integer index limit, falling back when unavailable or invalid.""" + try: + value = getattr(tag_index, name) + except (AttributeError, TypeError, ValueError): + return fallback + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + return fallback + return value + def _targets(self) -> tuple[list[_TagTarget], list[dict[str, str]]]: """Validate, normalize, and de-duplicate added/modified Markdown changes.""" assert self.context is not None @@ -94,10 +119,14 @@ class AutoTagStep(BaseStep): targets[path] = _TagTarget(change=normalized_change, path=path) return list(targets.values()), ignored - async def _process_target(self, target: _TagTarget, tag_key: str) -> str: + async def _process_target(self, target: _TagTarget, tag_key: str, max_tag_length: int) -> str: result = await self.agent_wrapper.reply( self.prompt_format("user_message", path=target.path, change=target.change, tag_key=tag_key), - system_prompt=self.prompt_format("system_prompt", tag_key=tag_key), + system_prompt=self.prompt_format( + "system_prompt", + tag_key=tag_key, + max_tags_per_file=self.max_tags_per_file, + ), job_tools=self.tools, injected_job_kwargs={ "_allowed_paths": [target.path], @@ -107,7 +136,11 @@ class AutoTagStep(BaseStep): path = Path(self.file_store.workspace_path or ".") / target.path metadata = dict(frontmatter.loads(path.read_text(encoding="utf-8")).metadata or {}) - normalized = normalize_memory_tags(metadata.get(tag_key)) + normalized = normalize_memory_tags( + metadata.get(tag_key), + max_tags_per_file=self.max_tags_per_file, + max_tag_length=max_tag_length, + ) if metadata.get(tag_key) != normalized: response = await self.run_job( "frontmatter_update", @@ -148,12 +181,23 @@ class AutoTagStep(BaseStep): return self.context.response tag_index = self.file_store.require_tag_index() if targets else None + max_tag_length = self._index_limit(tag_index, "max_tag_length", _DEFAULT_MAX_MEMORY_TAG_LENGTH) + index_max_tags = self._index_limit(tag_index, "max_tags_per_file", None) + if index_max_tags is not None and self.max_tags_per_file > index_max_tags: + self.context.response.success = False + if initial_success: + self.context.response.answer = ( + f"Error: auto_tag max_tags_per_file ({self.max_tags_per_file}) exceeds " + f"tag index limit ({index_max_tags})" + ) + return self.context.response + dates: set[str] = set() for target in targets: if day := self._daily_date(target.path): dates.add(day) try: - summary = await self._process_target(target, tag_index.tag_key) + summary = await self._process_target(target, tag_index.tag_key, max_tag_length) results.append( {"change": target.change, "path": target.path, "success": True, "summary": summary}, ) diff --git a/reme/steps/evolve/auto_tag.yaml b/reme/steps/evolve/auto_tag.yaml index 3c45aa60..0b3f5e45 100644 --- a/reme/steps/evolve/auto_tag.yaml +++ b/reme/steps/evolve/auto_tag.yaml @@ -6,9 +6,9 @@ system_prompt: | Required workflow: 1. Use `read` and `frontmatter_read` to inspect the complete target document and its current frontmatter. 2. Use `list_tags` to inspect tags already used in the workspace. Check additional pages when useful. - 3. Prefer one primary entity that best answers the question. Use two or three tags only when the memory is genuinely - about multiple independent central entities, and never use more than three. Prefer an existing tag whenever it - represents the same entity. Tags must not contain spaces; replace spaces with underscores, for example + 3. Prefer one primary entity that best answers the question. Use multiple tags only when the memory is genuinely + about multiple independent central entities, and never use more than {max_tags_per_file}. Prefer an existing tag + whenever it represents the same entity. Tags must not contain spaces; replace spaces with underscores, for example `sam_altman`. Examples include a person (`sam_altman`), an organization (`OpenAI`), or an asset/entity (`gold`). 4. Use `frontmatter_update` to write the complete `{tag_key}` list, including `[]` when no entity is central. @@ -25,9 +25,9 @@ system_prompt_zh: | 1. 使用 `read` 和 `frontmatter_read` 阅读目标文档的完整内容及当前 frontmatter。 2. 使用 `list_tags` 查看工作区已经使用的标签；必要时继续查看后续分页。 3. 默认只选择一个最能回答上述问题的主实体。只有当记忆确实同时围绕多个相互独立的核心实体时，才选择 - 两个或三个标签，并且绝不能超过三个。同一实体已有标签时必须优先复用。标签中不能包含空格；如有空格， - 必须替换为下划线，例如将 `Sam Altman` 写成 `sam_altman`。实体可以是人物（如 `sam_altman`）、组织或 - 公司（如 `宁德时代`），也可以是资产等实体（如 `黄金`）。 + 多个标签，并且绝不能超过 {max_tags_per_file} 个。同一实体已有标签时必须优先复用。标签中不能包含空格； + 如有空格，必须替换为下划线，例如将 `Sam Altman` 写成 `sam_altman`。实体可以是人物（如 `sam_altman`）、 + 组织或公司（如 `宁德时代`），也可以是资产等实体（如 `黄金`）。 4. 使用 `frontmatter_update` 写入完整的 `{tag_key}` 列表；没有核心实体时也要写入 `[]`。 只选择实体，不要使用宽泛主题、行为、属性、日期或普通关键词。每个实体使用一个简洁、规范的现实名称。 diff --git a/tests/unit/test_auto_tag.py b/tests/unit/test_auto_tag.py index f49a4e75..2fb3d8ba 100644 --- a/tests/unit/test_auto_tag.py +++ b/tests/unit/test_auto_tag.py @@ -180,13 +180,13 @@ async def test_auto_tag_uses_configured_key_and_normalizes_agent_output(tmp_path note = tmp_path / "memory/note.md" _write_note(note) store = LocalFileStore(name="store", embedding_store="", tag_index="") - store.tag_index = LocalTagIndex(tag_key="keywords") + store.tag_index = LocalTagIndex(tag_key="keywords", max_tag_length=8) wrapper = _TaggingWrapper( tmp_path, tag_key="keywords", tags=["OpenAI", "openai", "Sam Altman", "++", 100, "宁德时代", "黄金"], ) - step = AutoTagStep(file_store=store, agent_wrapper=wrapper) + step = AutoTagStep(file_store=store, agent_wrapper=wrapper, max_tags_per_file=2) async def update_frontmatter(name, /, **kwargs): assert name == "frontmatter_update" @@ -206,7 +206,6 @@ async def test_auto_tag_uses_configured_key_and_normalizes_agent_output(tmp_path assert response.answer == "Created memory/note.md" assert frontmatter.loads(note.read_text(encoding="utf-8")).metadata["keywords"] == [ "OpenAI", - "Sam Altman", "宁德时代", ] assert "Change: modified" in wrapper.calls[0][0] @@ -225,3 +224,27 @@ def test_normalize_memory_tags_enforces_entity_storage_contract(): "黄金", ], ) == ["OpenAI", "Sam Altman", "宁德时代"] + assert normalize_memory_tags( + ["one", "two", "three"], + max_tags_per_file=2, + max_tag_length=3, + ) == ["one", "two"] + + +@pytest.mark.asyncio +async def test_auto_tag_rejects_limit_above_tag_index_ceiling(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + note = tmp_path / "memory/note.md" + _write_note(note) + before = note.read_bytes() + store = LocalFileStore(name="store", embedding_store="", tag_index="") + store.tag_index = LocalTagIndex(max_tags_per_file=2) + wrapper = _TaggingWrapper(tmp_path) + step = AutoTagStep(file_store=store, agent_wrapper=wrapper, max_tags_per_file=3) + + response = await step(RuntimeContext(changes=[{"change": "modified", "path": "memory/note.md"}])) + + assert response.success is False + assert response.answer == "Error: auto_tag max_tags_per_file (3) exceeds tag index limit (2)" + assert not wrapper.calls + assert note.read_bytes() == before diff --git a/tests/unit/test_injected_job_kwargs.py b/tests/unit/test_injected_job_kwargs.py index a2c7cd02..a772f551 100644 --- a/tests/unit/test_injected_job_kwargs.py +++ b/tests/unit/test_injected_job_kwargs.py @@ -282,7 +282,7 @@ def test_configs_define_original_jobs_without_daily_variants(): default = resolve_app_config(config="default", log_config=False) assert default["jobs"]["auto_memory"]["steps"] == [ {"backend": "auto_memory_step"}, - {"backend": "auto_tag_step"}, + {"backend": "auto_tag_step", "max_tags_per_file": 3}, ] diff --git a/tests/unit/test_tag_index.py b/tests/unit/test_tag_index.py index f73099ef..821a33c6 100644 --- a/tests/unit/test_tag_index.py +++ b/tests/unit/test_tag_index.py @@ -570,5 +570,5 @@ def test_default_config_enables_tag_index_with_explicit_key() -> None: assert config["jobs"]["search"]["parameters"]["properties"]["tags"]["default"] == [] assert config["jobs"]["auto_memory"]["steps"] == [ {"backend": "auto_memory_step"}, - {"backend": "auto_tag_step"}, + {"backend": "auto_tag_step", "max_tags_per_file": 3}, ]