From 7e25d4679b7079522b0ed9662f065139de4c6d39 Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:52:52 +0800 Subject: [PATCH] feat: auto-tag plugin-generated reports (#534) * feat(plugins): auto-tag generated reports * fix(logging): forward host records on Python 3.13 * refactor(tags): decouple auto tagging from index updates * fix(tags): bind auto tagging to configured index * fix(tags): preserve standalone default index * fix(tags): make auto tagging best effort --- .github/workflows/release-auto-fin.yml | 4 +- .github/workflows/release-daily-paper.yml | 4 +- docs/en/plugin_management.md | 5 +- docs/zh/plugin_management.md | 4 +- plugins/auto-fin/README.md | 12 +- plugins/auto-fin/README_ZH.md | 13 +- plugins/auto-fin/pyproject.toml | 4 +- plugins/auto-fin/src/reme_auto_fin/merge.py | 3 + .../auto-fin/src/reme_auto_fin/plugin.yaml | 1 + plugins/auto-fin/tests/test_auto_fin.py | 3 + plugins/daily_paper/README.md | 16 +- plugins/daily_paper/README_ZH.md | 15 +- plugins/daily_paper/pyproject.toml | 4 +- .../src/reme_daily_paper/analyze.py | 11 + .../src/reme_daily_paper/digest.py | 8 +- .../src/reme_daily_paper/plugin.yaml | 1 + plugins/daily_paper/tests/test_daily_paper.py | 24 +- reme/__init__.py | 2 +- .../components/file_store/local_file_store.py | 7 +- reme/components/tag_index/base_tag_index.py | 17 +- reme/components/tag_index/local_tag_index.py | 18 +- reme/config/default.yaml | 2 - reme/constants.py | 5 + reme/steps/base_step.py | 10 +- reme/steps/evolve/auto_tag.py | 129 +++----- reme/utils/logger_utils.py | 10 +- tests/integration/test_auto_tag.py | 287 ++++++++++++++++++ tests/unit/test_auto_tag.py | 72 ++--- tests/unit/test_background_steps.py | 2 +- tests/unit/test_evolve_utils.py | 21 -- tests/unit/test_injected_job_kwargs.py | 4 +- tests/unit/test_package_versions.py | 12 +- tests/unit/test_tag_index.py | 99 +++++- 33 files changed, 604 insertions(+), 225 deletions(-) create mode 100644 tests/integration/test_auto_tag.py diff --git a/.github/workflows/release-auto-fin.yml b/.github/workflows/release-auto-fin.yml index 206a2b7a..d24e522d 100644 --- a/.github/workflows/release-auto-fin.yml +++ b/.github/workflows/release-auto-fin.yml @@ -75,8 +75,8 @@ jobs: reme_requirement = Requirement(requirements[0]) if reme_requirement.name != "reme-ai" or reme_requirement.extras: raise SystemExit(f"Expected a base reme-ai dependency, found {requirements[0]!r}") - if Version("0.4.1.8") in reme_requirement.specifier or Version("0.4.1.9") not in reme_requirement.specifier: - raise SystemExit(f"Expected reme-ai>=0.4.1.9, found {requirements[0]!r}") + if Version("0.4.1.11") in reme_requirement.specifier or Version("0.4.1.12") not in reme_requirement.specifier: + raise SystemExit(f"Expected reme-ai>=0.4.1.12, found {requirements[0]!r}") root_project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"] agentscope_requirements = [ Requirement(value) for value in root_project["optional-dependencies"]["as"] diff --git a/.github/workflows/release-daily-paper.yml b/.github/workflows/release-daily-paper.yml index e24a61df..13d86ceb 100644 --- a/.github/workflows/release-daily-paper.yml +++ b/.github/workflows/release-daily-paper.yml @@ -72,8 +72,8 @@ jobs: reme_requirements = [requirement for requirement in requirements if requirement.name == "reme-ai"] if len(reme_requirements) != 1 or reme_requirements[0].extras: raise SystemExit(f"Expected one base reme-ai dependency, found {reme_requirements!r}") - if Version("0.4.1.8") in reme_requirements[0].specifier or Version("0.4.1.9") not in reme_requirements[0].specifier: - raise SystemExit(f"Expected reme-ai>=0.4.1.9, found {reme_requirements!r}") + if Version("0.4.1.11") in reme_requirements[0].specifier or Version("0.4.1.12") not in reme_requirements[0].specifier: + raise SystemExit(f"Expected reme-ai>=0.4.1.12, found {reme_requirements!r}") if sum(requirement.name == "pypdf" for requirement in requirements) != 1: raise SystemExit("Expected exactly one pypdf dependency") root_project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"] diff --git a/docs/en/plugin_management.md b/docs/en/plugin_management.md index a2429787..4aea3810 100644 --- a/docs/en/plugin_management.md +++ b/docs/en/plugin_management.md @@ -173,8 +173,9 @@ curl -s http://127.0.0.1:2333/auto_fin \ When the application uses an MCP service, service-enabled plugin Jobs appear as MCP tools instead. -Custom application configs must provide the plugin's runtime dependencies, including an `agent_wrapper.default` and -the `search` and `read` Jobs used by Auto Fin. +Custom application configs must provide the plugin's runtime dependencies, including an `agent_wrapper.default`, a +`file_store.default` with an enabled tag index, and the `search`, `read`, `list_tags`, `frontmatter_read`, and +`frontmatter_update` Jobs used by Auto Fin and automatic tagging. ## Benchmark application presets diff --git a/docs/zh/plugin_management.md b/docs/zh/plugin_management.md index a0cdfde7..59fe934e 100644 --- a/docs/zh/plugin_management.md +++ b/docs/zh/plugin_management.md @@ -167,7 +167,9 @@ curl -s http://127.0.0.1:2333/auto_fin \ 当应用使用 MCP service 时,允许对外服务的插件 Job 会显示为 MCP tool。 -自定义应用配置需要提供插件的运行依赖,包括 `agent_wrapper.default`,以及 Auto Fin 使用的 `search` 和 `read` Jobs。 +自定义应用配置需要提供插件的运行依赖,包括 `agent_wrapper.default`、启用 tag index 的 +`file_store.default`,以及 Auto Fin 和自动标签使用的 `search`、`read`、`list_tags`、 +`frontmatter_read` 和 `frontmatter_update` Jobs。 ## Benchmark 应用配置 diff --git a/plugins/auto-fin/README.md b/plugins/auto-fin/README.md index 8012ce81..ba31c25e 100644 --- a/plugins/auto-fin/README.md +++ b/plugins/auto-fin/README.md @@ -17,7 +17,7 @@ through `plugins=["auto-fin"]`. ### 1. Install ReMe and Auto Fin ```bash -python -m pip install "reme-ai[core]>=0.4.1.9" +python -m pip install "reme-ai[core]>=0.4.1.12" reme plugins install reme-auto-fin ``` @@ -59,7 +59,9 @@ reme start plugins='["auto-fin"]' \ service.backend=http ``` -Custom application configs must provide `agent_wrapper.default` and the `search` and `read` Jobs used by Auto Fin. +Custom application configs must provide `agent_wrapper.default`, a `file_store.default` with an enabled tag index, and +the `search`, `read`, `list_tags`, `frontmatter_read`, and `frontmatter_update` Jobs used by Auto Fin and automatic +tagging. ## Pipeline @@ -75,6 +77,8 @@ research Agent uses search + read on historical memory validate historical wikilinks in code ↓ daily/YYYY-MM-DD/auto_fin.md + ↓ +generate memory tags; the background file watcher refreshes indexes ``` `auto_fin_data_step` signs and paginates the same endpoint used by the CLS website. It starts at the decision time and @@ -92,7 +96,9 @@ workspace-relative Markdown targets. Missing, absolute, escaping, backslash, and to their readable aliases. Same-day reruns use the existing report as context and replace it with the revised result. The final write is atomic and -refreshes the daily index. No JSONL, intermediate Markdown, or structured Agent output is written. +refreshes the daily index. The workflow then runs `auto_tag_step` to update the generated report's memory-tag +frontmatter; the normal background file watcher observes that source-file change and refreshes derived indexes. No +JSONL, intermediate Markdown, or structured Agent output is written. ## Parameters diff --git a/plugins/auto-fin/README_ZH.md b/plugins/auto-fin/README_ZH.md index e7811bb4..222c312a 100644 --- a/plugins/auto-fin/README_ZH.md +++ b/plugins/auto-fin/README_ZH.md @@ -14,7 +14,7 @@ distribution:单个 `reme.plugins` entry point 暴露 `plugin.yaml`,其中 ### 1. 安装 ReMe 和 Auto Fin ```bash -python -m pip install "reme-ai[core]>=0.4.1.9" +python -m pip install "reme-ai[core]>=0.4.1.12" reme plugins install reme-auto-fin ``` @@ -54,7 +54,9 @@ reme start plugins='["auto-fin"]' \ service.backend=http ``` -自定义应用配置需要提供 `agent_wrapper.default`,以及 Auto Fin 使用的 `search` 和 `read` Jobs。 +自定义应用配置需要提供 `agent_wrapper.default`、启用 tag index 的 `file_store.default`,以及 +Auto Fin 和自动标签使用的 `search`、`read`、`list_tags`、`frontmatter_read` 和 +`frontmatter_update` Jobs。 ## 流程 @@ -70,6 +72,8 @@ Research Agent 使用 search + read 检索历史记忆 代码校验历史 wikilink ↓ daily/YYYY-MM-DD/auto_fin.md + ↓ +生成记忆标签,由后台文件 watcher 刷新索引 ``` `auto_fin_data_step` 使用财联社网页同源接口的签名和分页方式,从分析时刻开始向前翻页,直到完整覆盖严格的最近 24 @@ -82,8 +86,9 @@ daily/YYYY-MM-DD/auto_fin.md Prompt 要求 Agent 只链接实际使用过的历史 Markdown;代码边界则独立保证只保留真实存在、相对 workspace 的 Markdown 目标。不存在、绝对路径、越界、带反斜杠和自引用的目标都会降级为可读 alias。 -同日重跑会参考当天已有报告并覆盖为修订结果。最终写入使用原子替换并刷新当天索引;流程不会写入 JSONL、中间 Markdown 或 Agent -结构化输出。 +同日重跑会参考当天已有报告并覆盖为修订结果。最终写入使用原子替换并刷新当天索引,随后通过 `auto_tag_step` +更新报告的记忆标签 frontmatter;常规后台文件 watcher 会观察该源文件变化并刷新派生索引。流程不会写入 JSONL、 +中间 Markdown 或 Agent 结构化输出。 ## 参数 diff --git a/plugins/auto-fin/pyproject.toml b/plugins/auto-fin/pyproject.toml index f09d4ea0..87107058 100644 --- a/plugins/auto-fin/pyproject.toml +++ b/plugins/auto-fin/pyproject.toml @@ -1,13 +1,13 @@ [project] name = "reme-auto-fin" -version = "0.1.2" +version = "0.1.3" description = "Auto Fin example plugin for ReMe." readme = "README.md" license = "Apache-2.0" license-files = ["LICENSE"] requires-python = ">=3.11" dependencies = [ - "reme-ai>=0.4.1.9", + "reme-ai>=0.4.1.12", ] [project.entry-points."reme.plugins"] diff --git a/plugins/auto-fin/src/reme_auto_fin/merge.py b/plugins/auto-fin/src/reme_auto_fin/merge.py index 2b5b8d1f..124bb934 100644 --- a/plugins/auto-fin/src/reme_auto_fin/merge.py +++ b/plugins/auto-fin/src/reme_auto_fin/merge.py @@ -104,6 +104,7 @@ class AutoFinMergeStep(AutoFinStep): async def execute(self): """Research the selected news and persist the validated report.""" assert self.context is not None + self.context["changes"] = [] if self.context.get("auto_fin_skipped"): return self.context.response run_date = date.fromisoformat(str(self._required("auto_fin_date"))) @@ -131,6 +132,7 @@ class AutoFinMergeStep(AutoFinStep): markdown = f"# {output.title}\n\n> {output.description}\n\n{output.body}\n\n" markdown += "> 未接入可靠行情数据;本文只提供新闻研究和回顾线索,不提供收益、目标价或买卖建议。\n" report = self._report_path(run_date) + change = "modified" if report.is_file() else "added" _write(report, markdown) await refresh_day_index( SimpleNamespace(workspace_path=self.workspace_path), @@ -138,6 +140,7 @@ class AutoFinMergeStep(AutoFinStep): str(self.config_value("daily_dir")), ) relative = report.relative_to(self.workspace_path).as_posix() + self.context["changes"] = [{"change": change, "path": relative}] self.context["markdown_path"] = relative self.context["auto_fin_digest_path"] = relative self.context.response.answer = output.body diff --git a/plugins/auto-fin/src/reme_auto_fin/plugin.yaml b/plugins/auto-fin/src/reme_auto_fin/plugin.yaml index 8edb8f81..67c29516 100644 --- a/plugins/auto-fin/src/reme_auto_fin/plugin.yaml +++ b/plugins/auto-fin/src/reme_auto_fin/plugin.yaml @@ -43,6 +43,7 @@ application_defaults: - backend: auto_fin_topic_step - backend: auto_fin_merge_step job_tools: [search, read] + - backend: auto_tag_step auto_fin_cron: backend: cron diff --git a/plugins/auto-fin/tests/test_auto_fin.py b/plugins/auto-fin/tests/test_auto_fin.py index 2815dd91..a273d68d 100644 --- a/plugins/auto-fin/tests/test_auto_fin.py +++ b/plugins/auto-fin/tests/test_auto_fin.py @@ -219,6 +219,7 @@ async def test_merge_writes_only_final_report_and_validates_historical_links(tmp assert "missing.md" not in report and "outside.md" not in report assert not (tmp_path / "daily" / "2026-08-10" / "auto_fin_news.md").exists() assert not (tmp_path / "resource").exists() + assert context["changes"] == [{"change": "added", "path": "daily/2026-08-10/auto_fin.md"}] assert response.metadata["source_paths"] == ["daily/2026-08-01/auto_fin.md"] @@ -262,8 +263,10 @@ def test_plugin_config_has_default_topics_and_no_intermediate_index_step(): "auto_fin_data_step", "auto_fin_topic_step", "auto_fin_merge_step", + "auto_tag_step", ] assert job["steps"][2]["job_tools"] == ["search", "read"] + assert job["steps"][3] == {"backend": "auto_tag_step"} assert jobs["auto_fin_cron"]["cron"] == "0 18 * * *" assert jobs["auto_fin_cron"]["steps"] == job["steps"] assert ( diff --git a/plugins/daily_paper/README.md b/plugins/daily_paper/README.md index d7915c11..2d00849d 100644 --- a/plugins/daily_paper/README.md +++ b/plugins/daily_paper/README.md @@ -13,7 +13,7 @@ their Job configuration under `application_defaults`. Enable the installed plugi ### 1. Install ReMe and Daily Paper ```bash -python -m pip install "reme-ai[core]>=0.4.1.9" +python -m pip install "reme-ai[core]>=0.4.1.12" reme plugins install reme-daily-paper ``` @@ -51,6 +51,10 @@ To run the Job once without starting a long-lived service: reme start plugins='["daily-paper"]' job=daily_paper topics="Agent memory" ``` +Custom application configs must provide `agent_wrapper.default`, a `file_store.default` with an enabled tag index, and +the `search`, `read`, `list_tags`, `frontmatter_read`, and `frontmatter_update` Jobs used by Daily Paper and automatic +tagging. + ## Pipeline ```text @@ -64,7 +68,9 @@ download and parse arXiv PDFs, then write three Chinese analyses ↓ use search + read to connect prior memory and generate a brief ↓ -refresh the daily index and optionally send the brief to DingTalk +generate memory tags; the background file watcher refreshes indexes + ↓ +optionally send the brief to DingTalk ``` `daily_paper_collect_step` concurrently reads the weekly and monthly rankings for the run date plus the strictly @@ -81,8 +87,10 @@ PDFs and files without a text layer fail explicitly. `daily_paper_digest_step` treats those three analyses as the factual source and receives only the read-only `search` and `read` tools for linking earlier memory. Code validates historical wikilinks, appends links to all -three source notes, and rebuilds the daily index. The optional `dingtalk_markdown_send_step` sends the final brief when -conversation IDs are configured and otherwise skips without side effects. +three source notes, and rebuilds the daily index. The workflow then runs `auto_tag_step` to update the memory-tag +frontmatter of all three analyses and the final brief. The normal background file watcher observes those source-file +changes and refreshes derived indexes before the optional `dingtalk_markdown_send_step` sends the brief. DingTalk +delivery skips without side effects when conversation IDs are not configured. ## Parameters diff --git a/plugins/daily_paper/README_ZH.md b/plugins/daily_paper/README_ZH.md index 08d11bbd..6fd8c5ab 100644 --- a/plugins/daily_paper/README_ZH.md +++ b/plugins/daily_paper/README_ZH.md @@ -11,7 +11,7 @@ Step backend,并在 `application_defaults` 下提供 Job 配置;通过 `plug ### 1. 安装 ReMe 和每日论文插件 ```bash -python -m pip install "reme-ai[core]>=0.4.1.9" +python -m pip install "reme-ai[core]>=0.4.1.12" reme plugins install reme-daily-paper ``` @@ -47,6 +47,10 @@ curl -s http://127.0.0.1:2333/daily_paper \ reme start plugins='["daily-paper"]' job=daily_paper topics="Agent memory" ``` +自定义应用配置需要提供 `agent_wrapper.default`、启用 tag index 的 `file_store.default`,以及 +Daily Paper 和自动标签使用的 `search`、`read`、`list_tags`、`frontmatter_read` 和 +`frontmatter_update` Jobs。 + ## 流程 ```text @@ -60,7 +64,9 @@ RRF 排序后由 Agent 精选三篇 ↓ 使用 search + read 关联历史记忆并生成简报 ↓ -写入当日索引,并按需发送到钉钉 +生成记忆标签,由后台文件 watcher 刷新索引 + ↓ +按需发送到钉钉 ``` `daily_paper_collect_step` 并发读取运行日期所在周和所在月的榜单,以及严格前一日的 Daily Papers。候选按 arXiv ID @@ -73,8 +79,9 @@ RRF 排序后由 Agent 精选三篇 文本。三篇中文解读按精选顺序写入当天目录;扫描版或没有文本层的 PDF 会明确失败。 `daily_paper_digest_step` 以本次生成的三篇解读为事实来源,只开放只读的 `search` 和 `read` 来关联较早记忆。 -代码会校验历史 wikilink、追加三篇源笔记链接,并重建当日索引。可选的 `dingtalk_markdown_send_step` 在配置群会话后 -发送最终简报;未配置时无副作用跳过。 +代码会校验历史 wikilink、追加三篇源笔记链接,并重建当日索引。随后 `auto_tag_step` 会更新三篇解读及最终简报的 +记忆标签 frontmatter,常规后台文件 watcher 会观察这些源文件变化并刷新派生索引,再由可选的 +`dingtalk_markdown_send_step` 发送最终简报;未配置群会话时无副作用跳过。 ## 参数 diff --git a/plugins/daily_paper/pyproject.toml b/plugins/daily_paper/pyproject.toml index cbdd51a2..eb131365 100644 --- a/plugins/daily_paper/pyproject.toml +++ b/plugins/daily_paper/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "reme-daily-paper" -version = "0.1.2" +version = "0.1.3" description = "Daily Paper research and reading-note plugin for ReMe." readme = "README.md" license = "Apache-2.0" @@ -8,7 +8,7 @@ license-files = ["LICENSE"] requires-python = ">=3.11" dependencies = [ "pypdf>=5.0.0", - "reme-ai>=0.4.1.9", + "reme-ai>=0.4.1.12", ] [project.entry-points."reme.plugins"] diff --git a/plugins/daily_paper/src/reme_daily_paper/analyze.py b/plugins/daily_paper/src/reme_daily_paper/analyze.py index 6f562e3e..a56ccddd 100644 --- a/plugins/daily_paper/src/reme_daily_paper/analyze.py +++ b/plugins/daily_paper/src/reme_daily_paper/analyze.py @@ -155,8 +155,18 @@ class DailyPaperAnalyzeStep(DailyPaperStep): "pdf_text_truncated": truncated, }, ) + changes = list(self.context.get("changes") or []) if existing_note is not None and existing_note != note_path: + existing_rel = existing_note.relative_to(self.workspace_path).as_posix() existing_note.unlink() + changes.append({"change": "deleted", "path": existing_rel}) + changes.append( + { + "change": "modified" if existing_note == note_path else "added", + "path": note_rel, + }, + ) + self.context["changes"] = changes self.logger.info( f"[{self.name}] paper done arxiv_id={paper.arxiv_id} note_path={note_rel}", ) @@ -172,6 +182,7 @@ class DailyPaperAnalyzeStep(DailyPaperStep): async def execute(self): assert self.context is not None + self.context["changes"] = [] if self._skip(): self.logger.info(f"[{self.name}] skip existing digest") return self.context.response diff --git a/plugins/daily_paper/src/reme_daily_paper/digest.py b/plugins/daily_paper/src/reme_daily_paper/digest.py index 121ee8f6..09484b37 100644 --- a/plugins/daily_paper/src/reme_daily_paper/digest.py +++ b/plugins/daily_paper/src/reme_daily_paper/digest.py @@ -108,6 +108,7 @@ class DailyPaperDigestStep(DailyPaperStep): title = normalize_chinese_title(output.title, f"每日论文简报-{day}") existing_rel = str(self._state("existing_digest_path") or "").strip() existing_path = self.workspace_path / existing_rel if existing_rel else None + existing_was_file = existing_path is not None and existing_path.is_file() title, digest_path = resolve_unique_note_path( self.workspace_path / daily_dir / day, title, @@ -135,8 +136,13 @@ class DailyPaperDigestStep(DailyPaperStep): "generated_at": utc_now_iso(), }, ) - if existing_path is not None and existing_path != digest_path: + changes = list(self.context.get("changes") or []) + if existing_was_file and existing_path != digest_path: existing_path.unlink() + changes.append({"change": "deleted", "path": existing_rel}) + digest_change = "modified" if existing_was_file and existing_path == digest_path else "added" + changes.append({"change": digest_change, "path": digest_rel}) + self.context["changes"] = changes self._set_state("digest_path", digest_rel) self.logger.info(f"[{self.name}] digest written path={digest_rel}") self.logger.info( diff --git a/plugins/daily_paper/src/reme_daily_paper/plugin.yaml b/plugins/daily_paper/src/reme_daily_paper/plugin.yaml index d8e62256..84e07350 100644 --- a/plugins/daily_paper/src/reme_daily_paper/plugin.yaml +++ b/plugins/daily_paper/src/reme_daily_paper/plugin.yaml @@ -54,6 +54,7 @@ application_defaults: - backend: daily_paper_analyze_step - backend: daily_paper_digest_step job_tools: [search, read] + - backend: auto_tag_step - backend: dingtalk_markdown_send_step input_mapping: daily_paper_digest_path: markdown_path diff --git a/plugins/daily_paper/tests/test_daily_paper.py b/plugins/daily_paper/tests/test_daily_paper.py index dcf6ecc6..8c9e46cc 100644 --- a/plugins/daily_paper/tests/test_daily_paper.py +++ b/plugins/daily_paper/tests/test_daily_paper.py @@ -63,7 +63,19 @@ def test_plugin_manifest_declares_complete_runtime_surface(): "daily_paper_analyze_step", "daily_paper_digest_step", } - assert set(_plugin_config()["jobs"]) == {"daily_paper", "daily_paper_cron"} + jobs = _plugin_config()["jobs"] + assert set(jobs) == {"daily_paper", "daily_paper_cron"} + assert [step["backend"] for step in jobs["daily_paper"]["steps"]] == [ + "daily_paper_collect_step", + "daily_paper_rank_step", + "daily_paper_select_step", + "daily_paper_analyze_step", + "daily_paper_digest_step", + "auto_tag_step", + "dingtalk_markdown_send_step", + ] + assert jobs["daily_paper"]["steps"][5] == {"backend": "auto_tag_step"} + assert jobs["daily_paper_cron"]["steps"] == jobs["daily_paper"]["steps"] class _QueuedAgentWrapper(BaseAgentWrapper): @@ -898,6 +910,12 @@ async def test_pipeline_filters_strict_yesterday_and_writes_outputs( "Clear evidence", ] assert digest.metadata["arxiv_ids"] == ["2607.10001", "2607.10004", "2607.10005"] + assert context["changes"] == [ + {"change": "added", "path": "daily/2026-07-21/记忆代理研究.md"}, + {"change": "added", "path": "daily/2026-07-21/上下文压缩研究.md"}, + {"change": "added", "path": "daily/2026-07-21/持续学习研究.md"}, + {"change": "added", "path": "daily/2026-07-21/今日智能体论文简报.md"}, + ] assert cc_wrapper.calls[0]["kwargs"] == {"output_schema": PaperPickList} assert "用户感兴趣的主题" not in cc_wrapper.calls[0]["inputs"] assert "用户未提供明确的 topic 倾向" in cc_wrapper.calls[0]["inputs"] @@ -984,6 +1002,10 @@ async def test_digest_force_migrates_old_fixed_filename_to_chinese_title(tmp_pat assert new_path.is_file() assert not old_path.exists() assert context["daily_paper_digest_path"] == "daily/2026-07-21/全新论文简报.md" + assert context["changes"] == [ + {"change": "deleted", "path": "daily/2026-07-21/daily-paper-brief.md"}, + {"change": "added", "path": "daily/2026-07-21/全新论文简报.md"}, + ] @pytest.mark.asyncio diff --git a/reme/__init__.py b/reme/__init__.py index 6ece0f6e..e8c66c1a 100644 --- a/reme/__init__.py +++ b/reme/__init__.py @@ -1,6 +1,6 @@ """ReMe CLI package.""" -__version__ = "0.4.1.11" +__version__ = "0.4.1.12" from . import config from . import constants diff --git a/reme/components/file_store/local_file_store.py b/reme/components/file_store/local_file_store.py index bccd8fbf..243917f8 100644 --- a/reme/components/file_store/local_file_store.py +++ b/reme/components/file_store/local_file_store.py @@ -66,7 +66,12 @@ class LocalFileStore(BaseFileStore): self.embedding_store = self.bind(embedding_store, BaseEmbeddingStore, default_factory=LocalEmbeddingStore) self.keyword_index = self.bind(keyword_index, BaseKeywordIndex, default_factory=BM25Index) self.file_graph = self.bind(file_graph, BaseFileGraph, default_factory=LocalFileGraph) - self.tag_index = self.bind(tag_index, BaseTagIndex, default_factory=LocalTagIndex) + self.tag_index = self.bind( + tag_index, + BaseTagIndex, + default_factory=LocalTagIndex if tag_index == "default" else None, + optional=False, + ) self.encoding = encoding self.store_version = store_version diff --git a/reme/components/tag_index/base_tag_index.py b/reme/components/tag_index/base_tag_index.py index 95882945..6f1acce6 100644 --- a/reme/components/tag_index/base_tag_index.py +++ b/reme/components/tag_index/base_tag_index.py @@ -4,6 +4,7 @@ from abc import abstractmethod from typing import ClassVar, Literal, TypedDict from ..base_component import BaseComponent +from ...constants import DEFAULT_MAX_MEMORY_TAG_LENGTH, DEFAULT_MAX_MEMORY_TAGS, DEFAULT_MEMORY_TAG_KEY from ...enumeration import ComponentEnum from ...schema import FileNode @@ -28,11 +29,25 @@ class BaseTagIndex(BaseComponent): component_type = ComponentEnum.TAG_INDEX reserved_tag_keys: ClassVar[frozenset[str]] = frozenset() - def __init__(self, tag_key: object = "memory_tags", **kwargs): + def __init__( + self, + tag_key: object = DEFAULT_MEMORY_TAG_KEY, + max_tags_per_file: int = DEFAULT_MAX_MEMORY_TAGS, + max_tag_length: int = DEFAULT_MAX_MEMORY_TAG_LENGTH, + **kwargs, + ): super().__init__(**kwargs) self._tag_key = self._validate_tag_key(tag_key) + self.max_tags_per_file = self._positive_int("max_tags_per_file", max_tags_per_file) + self.max_tag_length = self._positive_int("max_tag_length", max_tag_length) self.is_healthy = True + @staticmethod + def _positive_int(name: str, value: object) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + return value + @property def tag_key(self) -> str: """Frontmatter field from which this index derives tags.""" diff --git a/reme/components/tag_index/local_tag_index.py b/reme/components/tag_index/local_tag_index.py index e2f4e209..7392d668 100644 --- a/reme/components/tag_index/local_tag_index.py +++ b/reme/components/tag_index/local_tag_index.py @@ -20,16 +20,8 @@ class LocalTagIndex(BaseTagIndex): "status", } - def __init__( - self, - tag_key: object = "memory_tags", - max_tags_per_file: int = 3, - max_tag_length: int = 64, - **kwargs, - ): - super().__init__(tag_key=tag_key, **kwargs) - self.max_tags_per_file = self._positive_int("max_tags_per_file", max_tags_per_file) - self.max_tag_length = self._positive_int("max_tag_length", max_tag_length) + def __init__(self, **kwargs): + super().__init__(**kwargs) self.path_to_tags: dict[str, tuple[str, ...]] = {} self.tag_to_paths: dict[str, set[str]] = {} self._maintenance_lock = asyncio.Lock() @@ -38,12 +30,6 @@ class LocalTagIndex(BaseTagIndex): def n_files(self) -> int: return len(self.path_to_tags) - @staticmethod - def _positive_int(name: str, value: object) -> int: - if isinstance(value, bool) or not isinstance(value, int) or value <= 0: - raise ValueError(f"{name} must be a positive integer") - return value - def _normalize_tags(self, value: object, *, limit: int | None) -> list[str]: """Normalize a strict tag list, optionally limiting the result count.""" if not isinstance(value, list): diff --git a/reme/config/default.yaml b/reme/config/default.yaml index 9f62ccb8..3b7753a0 100644 --- a/reme/config/default.yaml +++ b/reme/config/default.yaml @@ -189,7 +189,6 @@ jobs: steps: - backend: auto_memory_step - backend: auto_tag_step - max_tags_per_file: 3 auto_memory_cc: backend: base @@ -208,7 +207,6 @@ jobs: steps: - backend: auto_memory_cc_step - backend: auto_tag_step - max_tags_per_file: 3 auto_resource: backend: base diff --git a/reme/constants.py b/reme/constants.py index c9566a7a..d58a3986 100644 --- a/reme/constants.py +++ b/reme/constants.py @@ -18,3 +18,8 @@ DEFAULT_MAX_IMAGE_BYTES = 5 * 1024 * 1024 # Background content-processing jobs skip files above this size. File watchers # and catalogs still track them so deletes and later size reductions are seen. DEFAULT_MAX_FILE_BYTES = 20 * 1024 * 1024 + +# Memory-tag generation and indexing defaults. +DEFAULT_MEMORY_TAG_KEY = "memory_tags" +DEFAULT_MAX_MEMORY_TAGS = 3 +DEFAULT_MAX_MEMORY_TAG_LENGTH = 64 diff --git a/reme/steps/base_step.py b/reme/steps/base_step.py index da9bce91..b2042237 100644 --- a/reme/steps/base_step.py +++ b/reme/steps/base_step.py @@ -71,12 +71,18 @@ class Ref: obj.__dict__.pop(self._cache_attr, None) def _resolve(self, obj: "BaseStep"): - for source in (obj.kwargs, obj.context or {}): + sources = (obj.kwargs, obj.context or {}) + for source in sources: value = source.get(self.key) if isinstance(value, self.base_cls): return value - name = obj.kwargs.get(self.key, "default") + name = "default" + for source in sources: + value = source.get(self.key) + if isinstance(value, str): + name = value + break if obj.app_context is None: if self.optional: return None diff --git a/reme/steps/evolve/auto_tag.py b/reme/steps/evolve/auto_tag.py index 19fe0dd5..05ea0604 100644 --- a/reme/steps/evolve/auto_tag.py +++ b/reme/steps/evolve/auto_tag.py @@ -1,20 +1,17 @@ """Generate entity-oriented memory tags for added or modified Markdown files.""" from dataclasses import dataclass -from pathlib import Path from typing import Literal import frontmatter from ._evolve import agent_reply_result_text from ..base_step import BaseStep -from ..file_io import parse_daily_date, refresh_day_index from ..file_io._path import display_path, resolve_path from ..index import normalize_posix_path from ...components import R +from ...constants import DEFAULT_MAX_MEMORY_TAG_LENGTH, DEFAULT_MAX_MEMORY_TAGS -_DEFAULT_MAX_MEMORY_TAGS = 3 -_DEFAULT_MAX_MEMORY_TAG_LENGTH = 64 _SUPPORTED_CHANGES = {"added", "modified"} @@ -24,21 +21,13 @@ class _TagTarget: path: str -def _positive_int(value: object, *, name: str) -> int: - if isinstance(value, bool) or not isinstance(value, int) or value <= 0: - raise ValueError(f"{name} must be a positive integer") - return value - - def normalize_memory_tags( value: object, *, - max_tags_per_file: int = _DEFAULT_MAX_MEMORY_TAGS, - max_tag_length: int = _DEFAULT_MAX_MEMORY_TAG_LENGTH, + max_tags_per_file: int = DEFAULT_MAX_MEMORY_TAGS, + max_tag_length: int = DEFAULT_MAX_MEMORY_TAG_LENGTH, ) -> list[str]: - """Normalize human-readable entity labels for frontmatter storage.""" - max_tags_per_file = _positive_int(max_tags_per_file, name="max_tags_per_file") - max_tag_length = _positive_int(max_tag_length, name="max_tag_length") + """Normalize source tags while preserving canonical display casing.""" if not isinstance(value, list): return [] @@ -64,22 +53,10 @@ def normalize_memory_tags( class AutoTagStep(BaseStep): """Update memory tags for Markdown files described by the common ``changes`` contract.""" - def __init__(self, max_tags_per_file: int = _DEFAULT_MAX_MEMORY_TAGS, **kwargs): + def __init__(self, **kwargs): super().__init__(**kwargs) - self.max_tags_per_file = _positive_int(max_tags_per_file, name="max_tags_per_file") self.tools = ["read", "list_tags", "frontmatter_read", "frontmatter_update"] - @staticmethod - def _index_limit(tag_index, name: str, fallback: int | None) -> int | None: - """Read one positive integer index limit, falling back when unavailable or invalid.""" - try: - value = getattr(tag_index, name) - except (AttributeError, TypeError, ValueError): - return fallback - if isinstance(value, bool) or not isinstance(value, int) or value <= 0: - return fallback - return value - def _targets(self) -> tuple[list[_TagTarget], list[dict[str, str]]]: """Validate, normalize, and de-duplicate added/modified Markdown changes.""" assert self.context is not None @@ -87,7 +64,7 @@ class AutoTagStep(BaseStep): if not isinstance(raw_changes, list): raise ValueError("AutoTagStep requires changes: list[dict]") - workspace = Path(self.file_store.workspace_path or ".").resolve() + workspace = self.workspace_path.resolve() targets: dict[str, _TagTarget] = {} ignored: list[dict[str, str]] = [] for item in raw_changes: @@ -98,7 +75,12 @@ class AutoTagStep(BaseStep): change = str(item.get("change") or "").strip().lower() raw_path = str(item.get("path") or "").strip() if change not in _SUPPORTED_CHANGES: - ignored.append({"path": raw_path, "reason": f"unsupported change: {change or 'missing'}"}) + ignored.append( + { + "path": raw_path, + "reason": f"unsupported change: {change or 'missing'}", + }, + ) continue target, error = resolve_path(workspace, raw_path) @@ -119,48 +101,44 @@ class AutoTagStep(BaseStep): targets[path] = _TagTarget(change=normalized_change, path=path) return list(targets.values()), ignored - async def _process_target(self, target: _TagTarget, tag_key: str, max_tag_length: int) -> str: + async def _process_target(self, target: _TagTarget) -> str: + file_store = self.file_store + tag_index = file_store.require_tag_index() + tag_key = tag_index.tag_key + tool_context = { + "file_store": file_store.name, + "_allowed_paths": [target.path], + "_allowed_frontmatter_keys": [tag_key], + } result = await self.agent_wrapper.reply( self.prompt_format("user_message", path=target.path, change=target.change, tag_key=tag_key), system_prompt=self.prompt_format( "system_prompt", tag_key=tag_key, - max_tags_per_file=self.max_tags_per_file, + max_tags_per_file=tag_index.max_tags_per_file, ), job_tools=self.tools, - injected_job_kwargs={ - "_allowed_paths": [target.path], - "_allowed_frontmatter_keys": [tag_key], - }, + injected_job_kwargs=tool_context, ) - path = Path(self.file_store.workspace_path or ".") / target.path + path = self.workspace_path / target.path metadata = dict(frontmatter.loads(path.read_text(encoding="utf-8")).metadata or {}) normalized = normalize_memory_tags( metadata.get(tag_key), - max_tags_per_file=self.max_tags_per_file, - max_tag_length=max_tag_length, + max_tags_per_file=tag_index.max_tags_per_file, + max_tag_length=tag_index.max_tag_length, ) if metadata.get(tag_key) != normalized: response = await self.run_job( "frontmatter_update", path=target.path, metadata={tag_key: normalized}, - _allowed_paths=[target.path], - _allowed_frontmatter_keys=[tag_key], + **tool_context, ) if not response.success: raise RuntimeError(str(response.answer)) return agent_reply_result_text(result) - def _daily_date(self, path: str) -> str | None: - daily_dir = normalize_posix_path(str(self.config_value("daily_dir"))).strip("/") - prefix = f"{daily_dir}/" - if not path.startswith(prefix): - return None - parts = path[len(prefix) :].split("/") - return parse_daily_date(parts[0]) if len(parts) == 2 else None - async def execute(self): assert self.context is not None initial_success = self.context.response.success @@ -173,54 +151,34 @@ class AutoTagStep(BaseStep): return self.context.response results: list[dict] = [] - indexes: list[dict] = [] - if targets and not self.file_store.tag_index_enabled: - self.context.response.success = False - if initial_success: - self.context.response.answer = "Error: tag index is not configured" - return self.context.response - - tag_index = self.file_store.require_tag_index() if targets else None - if tag_index is not None and not tag_index.is_healthy: - self.context.response.success = False - if initial_success: - self.context.response.answer = "Error: tag index unavailable" - return self.context.response - max_tag_length = self._index_limit(tag_index, "max_tag_length", _DEFAULT_MAX_MEMORY_TAG_LENGTH) - index_max_tags = self._index_limit(tag_index, "max_tags_per_file", None) - if index_max_tags is not None and self.max_tags_per_file > index_max_tags: - self.context.response.success = False - if initial_success: - self.context.response.answer = ( - f"Error: auto_tag max_tags_per_file ({self.max_tags_per_file}) exceeds " - f"tag index limit ({index_max_tags})" - ) - return self.context.response - - dates: set[str] = set() for target in targets: - if day := self._daily_date(target.path): - dates.add(day) try: - summary = await self._process_target(target, tag_index.tag_key, max_tag_length) + summary = await self._process_target(target) results.append( - {"change": target.change, "path": target.path, "success": True, "summary": summary}, + { + "change": target.change, + "path": target.path, + "success": True, + "summary": summary, + }, ) except Exception as exc: # pylint: disable=broad-exception-caught results.append( - {"change": target.change, "path": target.path, "success": False, "error": str(exc)}, + { + "change": target.change, + "path": target.path, + "success": False, + "error": str(exc), + }, ) self.logger.warning(f"[{self.name}] failed path={target.path}: {exc}") - for day in sorted(dates): - indexes.append(await refresh_day_index(self.file_store, day, self.config_value("daily_dir"))) - failed = sum(not item["success"] for item in results) succeeded = len(results) - failed - self.context.response.success = initial_success and failed == 0 - if initial_success and failed: + self.context.response.success = initial_success + if not initial_answer and failed: self.context.response.answer = f"Tagged {succeeded} file(s); {failed} failed" - elif initial_success and not initial_answer and succeeded: + elif not initial_answer and succeeded: self.context.response.answer = f"Tagged {succeeded} file(s)" else: self.context.response.answer = initial_answer @@ -230,6 +188,5 @@ class AutoTagStep(BaseStep): "failed": failed, "ignored": ignored, "results": results, - "indexes": indexes, } return self.context.response diff --git a/reme/utils/logger_utils.py b/reme/utils/logger_utils.py index 0cfe9ff7..ff87b58d 100644 --- a/reme/utils/logger_utils.py +++ b/reme/utils/logger_utils.py @@ -25,8 +25,14 @@ class _ForwardToLoggerHandler(logging.Handler): def emit(self, record: logging.LogRecord) -> None: target = logging.getLogger(self.target_name) - if target.isEnabledFor(record.levelno): - target.handle(record) + # Python 3.13 treats both ``isEnabledFor`` and ``handle`` as disabled + # during a nested logger call. Apply their stable public checks here, + # then dispatch directly so forwarded ReMe records are not dropped. + if target.disabled or target.manager.disable >= record.levelno or record.levelno < target.getEffectiveLevel(): + return + filtered = target.filter(record) + if filtered: + target.callHandlers(filtered if isinstance(filtered, logging.LogRecord) else record) class _QwenPawStdlibFormatter(logging.Formatter): diff --git a/tests/integration/test_auto_tag.py b/tests/integration/test_auto_tag.py new file mode 100644 index 00000000..3c154292 --- /dev/null +++ b/tests/integration/test_auto_tag.py @@ -0,0 +1,287 @@ +"""Real-LLM integration tests for ``auto_tag_step``. + +The fixtures mimic the final Markdown written by the Auto Fin and Daily Paper +plugins, then invoke AutoTagStep through a normal application Job. The tests +load the repository ``.env`` and therefore call the configured real LLM. +""" + +import asyncio +import sys +from pathlib import Path + +import frontmatter + +INTEGRATION_DIR = Path(__file__).resolve().parent +REPOSITORY = INTEGRATION_DIR.parents[1] +sys.path.insert(0, str(REPOSITORY)) +sys.path.insert(0, str(INTEGRATION_DIR)) + +# pylint: disable=wrong-import-position +from _workspace_fixture import workspace_env # noqa: E402 + +from reme.utils import load_env # noqa: E402 + +AUTO_TAG_JOB = { + "integration_auto_tag": { + "backend": "base", + "enable_serve": False, + "steps": [{"backend": "auto_tag_step"}], + }, +} + +AUTO_FIN_REPORT = """\ +--- +name: auto_fin +description: 宁德时代海外电池业务与产能进展的盘后研究 +kind: auto-fin-brief +--- + +# 宁德时代:欧洲电池业务进入产能兑现期 + +> 本文围绕宁德时代这一家公司,复盘其欧洲电池工厂的量产进展与客户交付节奏。 + +## 核心变化 + +宁德时代确认匈牙利工厂首条电芯产线开始试生产。管理层称,后续爬坡速度仍取决于良率、当地供应链和 +客户认证。该进展可能缩短欧洲客户的交付半径,但资本开支和产能利用率仍是主要风险。 + +## 观察框架 + +后续应继续跟踪宁德时代的海外产能利用率、单位成本与订单兑现情况。本文只提供新闻研究和回顾线索, +不提供收益、目标价或买卖建议。 +""" + +DAILY_PAPER_NOTES = { + "daily/2026-09-10/openai-agent-eval.md": """\ +--- +name: openai-agent-eval +description: OpenAI 智能体可靠性评测论文解读 +kind: daily-paper-analysis +arxiv_id: "2609.10001" +--- + +# OpenAI 智能体可靠性评测 + +这篇论文以 OpenAI 为唯一机构研究对象,分析其智能体在长任务中的失败恢复机制。实验比较了重试预算、 +工具错误和上下文压缩对完成率的影响,并讨论 OpenAI 对可靠性评测的设计选择。 +""", + "daily/2026-09-10/anthropic-context.md": """\ +--- +name: anthropic-context +description: Anthropic 长上下文研究论文解读 +kind: daily-paper-analysis +arxiv_id: "2609.10002" +--- + +# Anthropic 长上下文研究 + +论文只研究 Anthropic 的长上下文模型。作者测试信息位于不同位置时的召回差异,并分析 Anthropic 模型的 +注意力退化现象;上下文压缩只是实验方法,不是本文要标记的现实实体。 +""", + "daily/2026-09-10/nvidia-blackwell.md": """\ +--- +name: nvidia-blackwell +description: NVIDIA Blackwell 训练系统论文解读 +kind: daily-paper-analysis +arxiv_id: "2609.10003" +--- + +# NVIDIA Blackwell 训练系统 + +本文的机构研究对象是 NVIDIA。论文评估 Blackwell 集群的大模型训练吞吐、故障恢复和互连扩展效率, +并给出 NVIDIA 在超大规模训练系统上的工程取舍。 +""", +} + +DAILY_PAPER_DIGEST = """\ +--- +name: 每日论文简报-2026-09-10 +description: OpenAI、Anthropic 与 NVIDIA 三项研究的每日论文简报 +kind: daily-paper-brief +--- + +# 每日论文简报:智能体、长上下文与训练系统 + +今天的三篇论文分别围绕三个相互独立的机构实体展开:OpenAI 的智能体可靠性评测、Anthropic 的长上下文 +研究,以及 NVIDIA 的 Blackwell 训练系统。这三家机构都是本期简报的并列核心,而非顺带提及。 + +## 来源 + +- [[daily/2026-09-10/openai-agent-eval.md]] +- [[daily/2026-09-10/anthropic-context.md]] +- [[daily/2026-09-10/nvidia-blackwell.md]] +""" + +GENERIC_TAGS = { + "ai", + "人工智能", + "金融", + "股票", + "论文", + "研究", + "智能体", + "长上下文", + "训练系统", + "电池", + "半导体", +} + + +def _write(workspace: Path, relative: str, content: str) -> Path: + target = workspace / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + return target + + +def _post(path: Path) -> frontmatter.Post: + return frontmatter.loads(path.read_text(encoding="utf-8")) + + +def _tags(path: Path) -> list[str]: + value = _post(path).metadata.get("memory_tags") + assert isinstance(value, list), f"memory_tags was not written as a list: {value!r}" + assert all(isinstance(item, str) and item.strip() for item in value) + assert len(value) <= 3 + assert not ({item.casefold() for item in value} & {item.casefold() for item in GENERIC_TAGS}) + return value + + +def _assert_tag(tags: list[str], expected: str) -> None: + assert expected.casefold() in {tag.casefold() for tag in tags}, f"expected entity {expected!r}, got {tags!r}" + + +def _assert_any_tag(tags: list[str], expected: set[str]) -> None: + actual = {tag.casefold() for tag in tags} + accepted = {tag.casefold() for tag in expected} + assert actual & accepted, f"expected one of {sorted(expected)!r}, got {tags!r}" + + +async def _run_tag_job(env, changes: list[dict[str, str]]): + app = await env.make_app(jobs=AUTO_TAG_JOB) + response = await app.run_job("integration_auto_tag", changes=changes) + assert response.success is True, f"auto-tag failed: {response.answer!r}; metadata={response.metadata!r}" + auto_tag = response.metadata["auto_tag"] + assert auto_tag["processed"] == len(changes) + assert auto_tag["succeeded"] == len(changes) + assert auto_tag["failed"] == 0 + assert auto_tag["ignored"] == [] + return app, response + + +def test_auto_tag_auto_fin_report_uses_company_entity(): + """An Auto Fin report should be tagged with its company, not broad finance topics.""" + + async def run(): + load_env(REPOSITORY / ".env") + with workspace_env(load_env_file=False) as env: + relative = "daily/2026-09-10/auto_fin.md" + path = _write(env.workspace_dir, relative, AUTO_FIN_REPORT) + before = _post(path) + try: + _app, _response = await _run_tag_job(env, [{"change": "added", "path": relative}]) + tags = _tags(path) + _assert_any_tag(tags, {"宁德时代", "CATL"}) + assert len(tags) == 1, f"single-entity report received extra tags: {tags!r}" + after = _post(path) + assert after.content == before.content + assert {key: value for key, value in after.metadata.items() if key != "memory_tags"} == before.metadata + finally: + await env.close_all() + + asyncio.run(run()) + + +def test_auto_tag_daily_paper_batch_tags_analyses_and_digest(): + """The three analyses get one entity each and the digest gets all three.""" + + async def run(): + load_env(REPOSITORY / ".env") + with workspace_env(load_env_file=False) as env: + paths = { + relative: _write(env.workspace_dir, relative, content) + for relative, content in DAILY_PAPER_NOTES.items() + } + digest_rel = "daily/2026-09-10/daily-paper-brief.md" + paths[digest_rel] = _write(env.workspace_dir, digest_rel, DAILY_PAPER_DIGEST) + before = {relative: _post(path) for relative, path in paths.items()} + changes = [{"change": "added", "path": relative} for relative in paths] + try: + _app, response = await _run_tag_job(env, changes) + expected = { + "daily/2026-09-10/openai-agent-eval.md": "OpenAI", + "daily/2026-09-10/anthropic-context.md": "Anthropic", + "daily/2026-09-10/nvidia-blackwell.md": "NVIDIA", + } + for relative, entity in expected.items(): + tags = _tags(paths[relative]) + assert len(tags) == 1, f"single-entity analysis received extra tags: {relative} -> {tags!r}" + _assert_tag(tags, entity) + + digest_tags = _tags(paths[digest_rel]) + assert {tag.casefold() for tag in digest_tags} == { + "openai", + "anthropic", + "nvidia", + } + for relative, path in paths.items(): + after = _post(path) + assert after.content == before[relative].content + assert {key: value for key, value in after.metadata.items() if key != "memory_tags"} == before[ + relative + ].metadata + auto_tag = response.metadata["auto_tag"] + assert [item["path"] for item in auto_tag["results"]] == list(paths) + finally: + await env.close_all() + + asyncio.run(run()) + + +def test_auto_tag_modified_note_reuses_existing_canonical_tag(): + """A modified note should reuse an existing workspace spelling for the same entity.""" + + async def run(): + load_env(REPOSITORY / ".env") + with workspace_env(load_env_file=False) as env: + _write( + env.workspace_dir, + "daily/2026-09-09/openai-history.md", + "---\nname: openai-history\nmemory_tags: [OpenAI]\n---\n# OpenAI 历史记录\n", + ) + relative = "daily/2026-09-10/openai-update.md" + target = _write( + env.workspace_dir, + relative, + """\ +--- +name: openai-update +description: OpenAI, Inc. 产品更新记录 +memory_tags: [大模型] +status: reviewed +--- + +# OpenAI, Inc. 产品更新 + +这份记忆只围绕 OpenAI 公司,记录其产品发布节奏。文中的“大模型”是技术类别,不是现实实体标签。 +""", + ) + body_before = _post(target).content + try: + _app, response = await _run_tag_job(env, [{"change": "modified", "path": relative}]) + tags = _tags(target) + assert [tag.casefold() for tag in tags] == ["openai"] + post = _post(target) + assert post.content == body_before + assert post.metadata["status"] == "reviewed" + assert response.metadata["auto_tag"]["results"][0]["change"] == "modified" + finally: + await env.close_all() + + asyncio.run(run()) + + +if __name__ == "__main__": + test_auto_tag_auto_fin_report_uses_company_entity() + test_auto_tag_daily_paper_batch_tags_analyses_and_digest() + test_auto_tag_modified_note_reuses_existing_canonical_tag() diff --git a/tests/unit/test_auto_tag.py b/tests/unit/test_auto_tag.py index 2255a816..3336a381 100644 --- a/tests/unit/test_auto_tag.py +++ b/tests/unit/test_auto_tag.py @@ -12,7 +12,7 @@ from reme.components.file_store import LocalFileStore from reme.components.runtime_context import RuntimeContext from reme.components.tag_index import LocalTagIndex from reme.schema import Response -from reme.steps.evolve.auto_tag import AutoTagStep, normalize_memory_tags +from reme.steps.evolve.auto_tag import AutoTagStep class _TaggingWrapper(BaseAgentWrapper): @@ -49,20 +49,25 @@ def _write_note(path: Path) -> None: path.write_text("---\nname: note\ndescription: useful note\n---\nbody\n", encoding="utf-8") +def _file_store(*, name: str = "default", **tag_index_kwargs) -> LocalFileStore: + store = LocalFileStore(name=name, embedding_store="", tag_index="") + store.tag_index = LocalTagIndex(**tag_index_kwargs) + return store + + @pytest.mark.asyncio -async def test_auto_tag_handles_noop_and_rejects_invalid_preconditions(tmp_path, monkeypatch): +async def test_auto_tag_handles_noop_invalid_changes_and_no_tag_index(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) wrapper = _TaggingWrapper(tmp_path) - unindexed_store = LocalFileStore(name="store", embedding_store="", tag_index="") context = RuntimeContext(changes=[]) context.response.answer = "Skipped: no messages" - response = await AutoTagStep(file_store=unindexed_store, agent_wrapper=wrapper)(context) + response = await AutoTagStep(agent_wrapper=wrapper)(context) assert response.success is True assert response.answer == "Skipped: no messages" assert response.metadata["auto_tag"]["processed"] == 0 - response = await AutoTagStep(file_store=unindexed_store, agent_wrapper=wrapper)( + response = await AutoTagStep(agent_wrapper=wrapper)( RuntimeContext(changes="daily/note.md"), ) assert response.success is False @@ -70,36 +75,16 @@ async def test_auto_tag_handles_noop_and_rejects_invalid_preconditions(tmp_path, note = tmp_path / "daily/2026-09-09/note.md" _write_note(note) - before = note.read_bytes() change = RuntimeContext(changes=[{"change": "added", "path": "daily/2026-09-09/note.md"}]) - response = await AutoTagStep(file_store=unindexed_store, agent_wrapper=wrapper)(change) - - assert response.success is False - assert response.answer == "Error: tag index is not configured" - assert not wrapper.calls - assert note.read_bytes() == before - - indexed_store = LocalFileStore(name="store", embedding_store="", tag_index="") - indexed_store.tag_index = LocalTagIndex(max_tags_per_file=2) - response = await AutoTagStep( - file_store=indexed_store, - agent_wrapper=wrapper, - max_tags_per_file=3, - )(RuntimeContext(changes=[{"change": "added", "path": "daily/2026-09-09/note.md"}])) - assert response.success is False - assert response.answer == "Error: auto_tag max_tags_per_file (3) exceeds tag index limit (2)" - assert not wrapper.calls - assert note.read_bytes() == before - - indexed_store.tag_index = LocalTagIndex() - indexed_store.tag_index.set_healthy(False) - response = await AutoTagStep(file_store=indexed_store, agent_wrapper=wrapper)( - RuntimeContext(changes=[{"change": "added", "path": "daily/2026-09-09/note.md"}]), + response = await AutoTagStep(agent_wrapper=wrapper, file_store=LocalFileStore(embedding_store="", tag_index=""))( + change, ) - assert response.success is False - assert response.answer == "Error: tag index unavailable" + + assert response.success is True + assert response.answer == "Tagged 0 file(s); 1 failed" assert not wrapper.calls - assert note.read_bytes() == before + assert "memory_tags" not in frontmatter.loads(note.read_text(encoding="utf-8")).metadata + assert response.metadata["auto_tag"]["results"][0]["error"] == "tag index is not configured" @pytest.mark.asyncio @@ -112,10 +97,8 @@ async def test_auto_tag_filters_paths_and_continues_after_one_file_fails(tmp_pat (tmp_path / "daily/2026-09-09/notes").mkdir() (tmp_path / "daily/2026-09-09/plain.txt").write_text("text", encoding="utf-8") - store = LocalFileStore(name="store", embedding_store="", tag_index="") - store.tag_index = LocalTagIndex() wrapper = _TaggingWrapper(tmp_path, fail_name="failed.md") - step = AutoTagStep(file_store=store, agent_wrapper=wrapper) + step = AutoTagStep(agent_wrapper=wrapper, file_store=_file_store()) context = RuntimeContext( changes=[ {"change": "modified", "path": "daily/2026-09-09/failed.md"}, @@ -126,16 +109,20 @@ async def test_auto_tag_filters_paths_and_continues_after_one_file_fails(tmp_pat {"change": "deleted", "path": "daily/2026-09-09/deleted.md"}, ], ) + context.response.answer = "Generated report" response = await step(context) - assert response.success is False + assert response.success is True + assert response.answer == "Generated report" assert [call[1]["injected_job_kwargs"] for call in wrapper.calls] == [ { + "file_store": "default", "_allowed_paths": ["daily/2026-09-09/failed.md"], "_allowed_frontmatter_keys": ["memory_tags"], }, { + "file_store": "default", "_allowed_paths": ["daily/2026-09-09/first.md"], "_allowed_frontmatter_keys": ["memory_tags"], }, @@ -168,7 +155,6 @@ async def test_auto_tag_filters_paths_and_continues_after_one_file_fails(tmp_pat "summary": "tagged daily/2026-09-09/first.md", }, ] - assert "memory_tags: ['宁德时代', '黄金']" in (tmp_path / "daily/2026-09-09.md").read_text(encoding="utf-8") @pytest.mark.asyncio @@ -176,18 +162,18 @@ async def test_auto_tag_uses_configured_key_and_normalizes_agent_output(tmp_path monkeypatch.chdir(tmp_path) note = tmp_path / "memory/note.md" _write_note(note) - store = LocalFileStore(name="store", embedding_store="", tag_index="") - store.tag_index = LocalTagIndex(tag_key="keywords", max_tag_length=8) wrapper = _TaggingWrapper( tmp_path, tag_key="keywords", tags=["OpenAI", "openai", "Sam Altman", "++", 100, "宁德时代", "黄金"], ) - step = AutoTagStep(file_store=store, agent_wrapper=wrapper, max_tags_per_file=2) + store = _file_store(name="archive", tag_key="keywords", max_tags_per_file=2, max_tag_length=8) + step = AutoTagStep(agent_wrapper=wrapper, file_store=store) async def update_frontmatter(name, /, **kwargs): assert name == "frontmatter_update" assert kwargs["_allowed_frontmatter_keys"] == ["keywords"] + assert kwargs["file_store"] == "archive" post = frontmatter.loads(note.read_text(encoding="utf-8")) post.metadata.update(kwargs["metadata"]) note.write_text(frontmatter.dumps(post), encoding="utf-8") @@ -205,8 +191,4 @@ async def test_auto_tag_uses_configured_key_and_normalizes_agent_output(tmp_path "OpenAI", "宁德时代", ] - assert normalize_memory_tags( - ["one", "two", "three"], - max_tags_per_file=2, - max_tag_length=3, - ) == ["one", "two"] + assert wrapper.calls[0][1]["injected_job_kwargs"]["file_store"] == "archive" diff --git a/tests/unit/test_background_steps.py b/tests/unit/test_background_steps.py index 9cdbfe22..5ab2677d 100644 --- a/tests/unit/test_background_steps.py +++ b/tests/unit/test_background_steps.py @@ -1998,7 +1998,7 @@ def test_auto_memory_reports_modified_for_create_and_false_for_skip(): with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir): cwd = Path.cwd() app_ctx = _make_app_context(cwd) - fs = LocalFileStore(name="test_store", embedding_store="", tag_index="default") + fs = LocalFileStore(name="test_store", embedding_store="", tag_index="") wrapper = _FakeAgentWrapper() await fs.start() _install_file_jobs(app_ctx, fs) diff --git a/tests/unit/test_evolve_utils.py b/tests/unit/test_evolve_utils.py index c29cd0f9..85d5ab69 100644 --- a/tests/unit/test_evolve_utils.py +++ b/tests/unit/test_evolve_utils.py @@ -10,7 +10,6 @@ import pytest from reme.steps.evolve._evolve import agent_reply_result_text, format_history from reme.steps.evolve.auto_memory import AutoMemoryStep, _sanitize_msg_for_save -from reme.steps.evolve.auto_tag import normalize_memory_tags def test_agent_reply_result_text_uses_last_text_block(): @@ -76,26 +75,6 @@ def test_sanitize_msg_for_save_drops_tool_results_and_base64_data(): assert sanitized.content[1].name == "memory_search" -def test_auto_tag_normalizes_frontmatter_tags(): - """Memory tags preserve entity names, de-duplicate, and stop at three.""" - assert normalize_memory_tags( - [ - "OpenAI", - "openai", - "Sam Altman", - "++", - 100, - "宁德时代", - "黄金", - ], - ) == ["OpenAI", "Sam_Altman", "宁德时代"] - # pylint: disable=use-implicit-booleaness-not-comparison - assert normalize_memory_tags(None) == [] - assert normalize_memory_tags("OpenAI") == [] - # pylint: enable=use-implicit-booleaness-not-comparison - assert normalize_memory_tags(["x" * 65, True, {}, "宁德时代"]) == ["宁德时代"] - - def test_auto_memory_accepts_message_timestamp_aliases(): """AutoMemoryStep preserves historical message timestamps from common benchmark fields.""" top_level = AutoMemoryStep._to_msg( diff --git a/tests/unit/test_injected_job_kwargs.py b/tests/unit/test_injected_job_kwargs.py index f4e32500..5da892e8 100644 --- a/tests/unit/test_injected_job_kwargs.py +++ b/tests/unit/test_injected_job_kwargs.py @@ -261,11 +261,11 @@ def test_configs_define_original_jobs_without_daily_variants(): default = resolve_app_config(config="default", log_config=False) assert default["jobs"]["auto_memory"]["steps"] == [ {"backend": "auto_memory_step"}, - {"backend": "auto_tag_step", "max_tags_per_file": 3}, + {"backend": "auto_tag_step"}, ] assert default["jobs"]["auto_memory_cc"]["steps"] == [ {"backend": "auto_memory_cc_step"}, - {"backend": "auto_tag_step", "max_tags_per_file": 3}, + {"backend": "auto_tag_step"}, ] diff --git a/tests/unit/test_package_versions.py b/tests/unit/test_package_versions.py index 5cd8a26e..85184660 100644 --- a/tests/unit/test_package_versions.py +++ b/tests/unit/test_package_versions.py @@ -56,8 +56,8 @@ def test_studio_packages_have_independent_identity() -> None: assert main_config["project"]["optional-dependencies"]["core"].count("reme-ai[as]") == 1 assert main_config["project"]["optional-dependencies"]["core"].count("reme_studio") == 1 assert "qwenpaw" not in main_config["project"]["optional-dependencies"] - assert auto_fin_config["project"]["version"] == "0.1.2" - assert daily_paper_config["project"]["version"] == "0.1.2" + assert auto_fin_config["project"]["version"] == "0.1.3" + assert daily_paper_config["project"]["version"] == "0.1.3" assert main_config["tool"]["setuptools"]["packages"]["find"]["include"] == ["reme", "reme.*"] assert "reme_studio*" in main_config["tool"]["setuptools"]["packages"]["find"]["exclude"] @@ -167,8 +167,8 @@ def test_auto_fin_requires_reme_base() -> None: assert len(reme_requirements) == 1 assert not reme_requirements[0].extras - assert Version("0.4.1.8") not in reme_requirements[0].specifier - assert Version("0.4.1.9") in reme_requirements[0].specifier + assert Version("0.4.1.11") not in reme_requirements[0].specifier + assert Version("0.4.1.12") in reme_requirements[0].specifier def test_daily_paper_license_matches_repository() -> None: @@ -185,8 +185,8 @@ def test_daily_paper_declares_runtime_dependencies() -> None: by_name = {requirement.name: requirement for requirement in requirements} assert not by_name["reme-ai"].extras - assert Version("0.4.1.8") not in by_name["reme-ai"].specifier - assert Version("0.4.1.9") in by_name["reme-ai"].specifier + assert Version("0.4.1.11") not in by_name["reme-ai"].specifier + assert Version("0.4.1.12") in by_name["reme-ai"].specifier assert "pypdf" in by_name diff --git a/tests/unit/test_tag_index.py b/tests/unit/test_tag_index.py index d4189493..845eb279 100644 --- a/tests/unit/test_tag_index.py +++ b/tests/unit/test_tag_index.py @@ -7,10 +7,12 @@ from pathlib import Path import pytest +from reme.components.application_context import ApplicationContext from reme.components.file_chunker import MarkdownFileChunker from reme.components.file_store import LocalFileStore from reme.components.tag_index import LocalTagIndex from reme.config import resolve_app_config +from reme.enumeration import ComponentEnum from reme.schema import FileChunk, FileFrontMatter, FileNode from reme.steps.index.list_tags import ListTagsStep @@ -24,6 +26,10 @@ def _chunk(chunk_id: str, path: str, text: str) -> FileChunk: return FileChunk(id=chunk_id, path=path, text=text, start_line=1, end_line=1) +def _standalone_file_store_with_tag_index(**kwargs) -> LocalFileStore: + return LocalFileStore(tag_index="default", **kwargs) + + def test_tag_normalization_and_bidirectional_mutations() -> None: """Normalize FileNode tags and keep both lookup directions consistent.""" @@ -209,6 +215,28 @@ def test_list_tags_paginates_and_applies_default_sort_orders() -> None: assert "empty page" in job["parameters"]["properties"]["page"]["description"] +@pytest.mark.asyncio +async def test_list_tags_resolves_named_file_store_from_runtime_context(tmp_path) -> None: + """Injected tool context can route list_tags to the AutoTag Step's file store.""" + context = ApplicationContext(workspace_dir=str(tmp_path)) + default_store = LocalFileStore(name="default", embedding_store="", tag_index="") + archive_store = LocalFileStore(name="archive", embedding_store="", tag_index="") + default_store.tag_index = LocalTagIndex() + archive_store.tag_index = LocalTagIndex() + await default_store.tag_index.rebuild([_node("daily/default.md", ["default-tag"])]) + await archive_store.tag_index.rebuild([_node("daily/archive.md", ["archive-tag"])]) + context.components = { + ComponentEnum.FILE_STORE: { + "default": default_store, + "archive": archive_store, + }, + } + + response = await ListTagsStep(app_context=context)(file_store="archive") + + assert response.answer["items"] == [("archive-tag", 1)] + + def test_configured_frontmatter_key_contract() -> None: """Validate, apply, and invalidate changes to the configured source key.""" @@ -259,6 +287,55 @@ def test_configured_frontmatter_key_contract() -> None: assert not index.is_healthy +def test_file_store_tag_index_binding_only_defaults_for_default_name() -> None: + """Support standalone defaults without silently substituting named dependencies.""" + default_store = LocalFileStore(name="default-store", embedding_store="", tag_index="default") + custom_store = LocalFileStore(name="custom-store", embedding_store="", tag_index="custom") + + default_dependency = default_store.dependency_bindings["tag_index"] + assert default_dependency.name == "default" + assert default_dependency.default_factory is LocalTagIndex + assert default_dependency.optional is False + + custom_dependency = custom_store.dependency_bindings["tag_index"] + assert custom_dependency.name == "custom" + assert custom_dependency.default_factory is None + assert custom_dependency.optional is False + + +def test_file_store_without_tag_index_name_disables_tag_index() -> None: + """Treat both an omitted tag-index setting and an explicit empty name as disabled.""" + omitted = LocalFileStore(name="omitted", embedding_store="") + explicit = LocalFileStore(name="explicit", embedding_store="", tag_index="") + + assert omitted.tag_index is None + assert omitted.tag_index_enabled is False + assert explicit.tag_index is None + assert explicit.tag_index_enabled is False + + +@pytest.mark.parametrize("tag_index_name", ["", "custom", "missing"]) +def test_file_store_resolves_configured_tag_index(tag_index_name: str, tmp_path: Path) -> None: + """Resolve tag indexes by configured component name and fail on missing names.""" + + async def run() -> None: + context = ApplicationContext(workspace_dir=str(tmp_path)) + index = LocalTagIndex(name="custom", tag_key="keywords") + context.components = {ComponentEnum.TAG_INDEX: {"custom": index}} + store = LocalFileStore(app_context=context, embedding_store="", tag_index=tag_index_name) + + if tag_index_name == "missing": + with pytest.raises(ValueError, match="tag_index 'missing' not found"): + await store.start() + assert not store.is_started + else: + await store._resolve_bindings() + assert store.tag_index is (index if tag_index_name else None) + assert store.tag_index_enabled is bool(tag_index_name) + + asyncio.run(run()) + + def test_file_store_updates_tag_index_from_file_nodes(monkeypatch, tmp_path: Path) -> None: """Keep daily and digest tags aligned through file-store mutations.""" @@ -269,7 +346,7 @@ def test_file_store_updates_tag_index_from_file_nodes(monkeypatch, tmp_path: Pat async def run() -> None: monkeypatch.chdir(tmp_path) - store = LocalFileStore(name="test", embedding_store="", tag_index="default") + store = _standalone_file_store_with_tag_index(name="test", embedding_store="") await store.start() assert isinstance(store.tag_index, LocalTagIndex) assert store.tag_index_enabled is True @@ -304,7 +381,7 @@ def test_tag_failures_do_not_block_other_indexes_and_retry_rebuild(monkeypatch, async def run() -> None: monkeypatch.chdir(tmp_path) - store = LocalFileStore(name="test", embedding_store="", tag_index="default") + store = _standalone_file_store_with_tag_index(name="test", embedding_store="") await store.start() assert store.tag_index_enabled original_rebuild = store.tag_index.rebuild @@ -344,7 +421,7 @@ def test_failed_tag_reconciliation_makes_queries_fail_closed(monkeypatch, tmp_pa async def run() -> None: monkeypatch.chdir(tmp_path) - store = LocalFileStore(name="test", embedding_store="", tag_index="default") + store = _standalone_file_store_with_tag_index(name="test", embedding_store="") await store.start() assert store.tag_index_enabled await store.upsert([(_node("daily/a.md", ["old"]), [])]) @@ -371,7 +448,7 @@ def test_tag_rebuild_graph_read_failure_does_not_block_upsert(monkeypatch, tmp_p async def run() -> None: monkeypatch.chdir(tmp_path) - store = LocalFileStore(name="test", embedding_store="", tag_index="default") + store = _standalone_file_store_with_tag_index(name="test", embedding_store="") await store.start() assert store.tag_index_enabled assert store.file_graph is not None @@ -401,7 +478,7 @@ def test_explicit_reindex_restores_tag_index(monkeypatch, tmp_path: Path) -> Non async def run() -> None: monkeypatch.chdir(tmp_path) - store = LocalFileStore(name="test", embedding_store="", tag_index="default") + store = _standalone_file_store_with_tag_index(name="test", embedding_store="") await store.start() assert store.tag_index_enabled await store.upsert( @@ -437,7 +514,7 @@ def test_tag_delete_failures_do_not_block_core_deletion(monkeypatch, tmp_path: P async def run() -> None: monkeypatch.chdir(tmp_path) - store = LocalFileStore(name="test", embedding_store="", tag_index="default") + store = _standalone_file_store_with_tag_index(name="test", embedding_store="") await store.start() assert store.tag_index_enabled chunk = _chunk("chunk-a", "daily/a.md", "alpha memory") @@ -472,7 +549,7 @@ def test_existing_markdown_chunker_supplies_frontmatter_tags(monkeypatch, tmp_pa note.write_text("---\nmemory_tags: [Python, ReMe]\n---\nbody\n", encoding="utf-8") node, chunks = await MarkdownFileChunker().chunk(note) - store = LocalFileStore(name="test", embedding_store="", tag_index="default") + store = _standalone_file_store_with_tag_index(name="test", embedding_store="") await store.start() await store.upsert([(node, chunks)]) @@ -487,14 +564,14 @@ def test_file_store_rebuilds_non_persistent_tag_index_from_graph(monkeypatch, tm async def run() -> None: monkeypatch.chdir(tmp_path) - first = LocalFileStore(name="test", embedding_store="", tag_index="default") + first = _standalone_file_store_with_tag_index(name="test", embedding_store="") await first.start() await first.upsert([(_node("daily/a.md", ["ReMe"]), [])]) await first.close() assert not list((tmp_path / "metadata").glob("tag_index/**/*")) - restored = LocalFileStore(name="test", embedding_store="", tag_index="default") + restored = _standalone_file_store_with_tag_index(name="test", embedding_store="") await restored.start() assert await restored.tag_index.paths_for_tags(["reme"]) == ["daily/a.md"] await restored.close() @@ -515,9 +592,9 @@ def test_default_config_enables_tag_index_with_explicit_key() -> None: assert config["jobs"]["search"]["parameters"]["properties"]["tags"]["default"] == [] assert config["jobs"]["auto_memory"]["steps"] == [ {"backend": "auto_memory_step"}, - {"backend": "auto_tag_step", "max_tags_per_file": 3}, + {"backend": "auto_tag_step"}, ] assert config["jobs"]["auto_memory_cc"]["steps"] == [ {"backend": "auto_memory_cc_step"}, - {"backend": "auto_tag_step", "max_tags_per_file": 3}, + {"backend": "auto_tag_step"}, ]