mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-11 22:51:10 +00:00
feat: auto-tag plugin-generated reports (#534)
* feat(plugins): auto-tag generated reports * fix(logging): forward host records on Python 3.13 * refactor(tags): decouple auto tagging from index updates * fix(tags): bind auto tagging to configured index * fix(tags): preserve standalone default index * fix(tags): make auto tagging best effort
This commit is contained in:
parent
9975bb37b9
commit
7e25d4679b
33 changed files with 604 additions and 225 deletions
4
.github/workflows/release-auto-fin.yml
vendored
4
.github/workflows/release-auto-fin.yml
vendored
|
|
@ -75,8 +75,8 @@ jobs:
|
|||
reme_requirement = Requirement(requirements[0])
|
||||
if reme_requirement.name != "reme-ai" or reme_requirement.extras:
|
||||
raise SystemExit(f"Expected a base reme-ai dependency, found {requirements[0]!r}")
|
||||
if Version("0.4.1.8") in reme_requirement.specifier or Version("0.4.1.9") not in reme_requirement.specifier:
|
||||
raise SystemExit(f"Expected reme-ai>=0.4.1.9, found {requirements[0]!r}")
|
||||
if Version("0.4.1.11") in reme_requirement.specifier or Version("0.4.1.12") not in reme_requirement.specifier:
|
||||
raise SystemExit(f"Expected reme-ai>=0.4.1.12, found {requirements[0]!r}")
|
||||
root_project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
|
||||
agentscope_requirements = [
|
||||
Requirement(value) for value in root_project["optional-dependencies"]["as"]
|
||||
|
|
|
|||
4
.github/workflows/release-daily-paper.yml
vendored
4
.github/workflows/release-daily-paper.yml
vendored
|
|
@ -72,8 +72,8 @@ jobs:
|
|||
reme_requirements = [requirement for requirement in requirements if requirement.name == "reme-ai"]
|
||||
if len(reme_requirements) != 1 or reme_requirements[0].extras:
|
||||
raise SystemExit(f"Expected one base reme-ai dependency, found {reme_requirements!r}")
|
||||
if Version("0.4.1.8") in reme_requirements[0].specifier or Version("0.4.1.9") not in reme_requirements[0].specifier:
|
||||
raise SystemExit(f"Expected reme-ai>=0.4.1.9, found {reme_requirements!r}")
|
||||
if Version("0.4.1.11") in reme_requirements[0].specifier or Version("0.4.1.12") not in reme_requirements[0].specifier:
|
||||
raise SystemExit(f"Expected reme-ai>=0.4.1.12, found {reme_requirements!r}")
|
||||
if sum(requirement.name == "pypdf" for requirement in requirements) != 1:
|
||||
raise SystemExit("Expected exactly one pypdf dependency")
|
||||
root_project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
|
||||
|
|
|
|||
|
|
@ -173,8 +173,9 @@ curl -s http://127.0.0.1:2333/auto_fin \
|
|||
|
||||
When the application uses an MCP service, service-enabled plugin Jobs appear as MCP tools instead.
|
||||
|
||||
Custom application configs must provide the plugin's runtime dependencies, including an `agent_wrapper.default` and
|
||||
the `search` and `read` Jobs used by Auto Fin.
|
||||
Custom application configs must provide the plugin's runtime dependencies, including an `agent_wrapper.default`, a
|
||||
`file_store.default` with an enabled tag index, and the `search`, `read`, `list_tags`, `frontmatter_read`, and
|
||||
`frontmatter_update` Jobs used by Auto Fin and automatic tagging.
|
||||
|
||||
## Benchmark application presets
|
||||
|
||||
|
|
|
|||
|
|
@ -167,7 +167,9 @@ curl -s http://127.0.0.1:2333/auto_fin \
|
|||
|
||||
当应用使用 MCP service 时,允许对外服务的插件 Job 会显示为 MCP tool。
|
||||
|
||||
自定义应用配置需要提供插件的运行依赖,包括 `agent_wrapper.default`,以及 Auto Fin 使用的 `search` 和 `read` Jobs。
|
||||
自定义应用配置需要提供插件的运行依赖,包括 `agent_wrapper.default`、启用 tag index 的
|
||||
`file_store.default`,以及 Auto Fin 和自动标签使用的 `search`、`read`、`list_tags`、
|
||||
`frontmatter_read` 和 `frontmatter_update` Jobs。
|
||||
|
||||
## Benchmark 应用配置
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ through `plugins=["auto-fin"]`.
|
|||
### 1. Install ReMe and Auto Fin
|
||||
|
||||
```bash
|
||||
python -m pip install "reme-ai[core]>=0.4.1.9"
|
||||
python -m pip install "reme-ai[core]>=0.4.1.12"
|
||||
reme plugins install reme-auto-fin
|
||||
```
|
||||
|
||||
|
|
@ -59,7 +59,9 @@ reme start plugins='["auto-fin"]' \
|
|||
service.backend=http
|
||||
```
|
||||
|
||||
Custom application configs must provide `agent_wrapper.default` and the `search` and `read` Jobs used by Auto Fin.
|
||||
Custom application configs must provide `agent_wrapper.default`, a `file_store.default` with an enabled tag index, and
|
||||
the `search`, `read`, `list_tags`, `frontmatter_read`, and `frontmatter_update` Jobs used by Auto Fin and automatic
|
||||
tagging.
|
||||
|
||||
## Pipeline
|
||||
|
||||
|
|
@ -75,6 +77,8 @@ research Agent uses search + read on historical memory
|
|||
validate historical wikilinks in code
|
||||
↓
|
||||
daily/YYYY-MM-DD/auto_fin.md
|
||||
↓
|
||||
generate memory tags; the background file watcher refreshes indexes
|
||||
```
|
||||
|
||||
`auto_fin_data_step` signs and paginates the same endpoint used by the CLS website. It starts at the decision time and
|
||||
|
|
@ -92,7 +96,9 @@ workspace-relative Markdown targets. Missing, absolute, escaping, backslash, and
|
|||
to their readable aliases.
|
||||
|
||||
Same-day reruns use the existing report as context and replace it with the revised result. The final write is atomic and
|
||||
refreshes the daily index. No JSONL, intermediate Markdown, or structured Agent output is written.
|
||||
refreshes the daily index. The workflow then runs `auto_tag_step` to update the generated report's memory-tag
|
||||
frontmatter; the normal background file watcher observes that source-file change and refreshes derived indexes. No
|
||||
JSONL, intermediate Markdown, or structured Agent output is written.
|
||||
|
||||
## Parameters
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ distribution:单个 `reme.plugins` entry point 暴露 `plugin.yaml`,其中
|
|||
### 1. 安装 ReMe 和 Auto Fin
|
||||
|
||||
```bash
|
||||
python -m pip install "reme-ai[core]>=0.4.1.9"
|
||||
python -m pip install "reme-ai[core]>=0.4.1.12"
|
||||
reme plugins install reme-auto-fin
|
||||
```
|
||||
|
||||
|
|
@ -54,7 +54,9 @@ reme start plugins='["auto-fin"]' \
|
|||
service.backend=http
|
||||
```
|
||||
|
||||
自定义应用配置需要提供 `agent_wrapper.default`,以及 Auto Fin 使用的 `search` 和 `read` Jobs。
|
||||
自定义应用配置需要提供 `agent_wrapper.default`、启用 tag index 的 `file_store.default`,以及
|
||||
Auto Fin 和自动标签使用的 `search`、`read`、`list_tags`、`frontmatter_read` 和
|
||||
`frontmatter_update` Jobs。
|
||||
|
||||
## 流程
|
||||
|
||||
|
|
@ -70,6 +72,8 @@ Research Agent 使用 search + read 检索历史记忆
|
|||
代码校验历史 wikilink
|
||||
↓
|
||||
daily/YYYY-MM-DD/auto_fin.md
|
||||
↓
|
||||
生成记忆标签,由后台文件 watcher 刷新索引
|
||||
```
|
||||
|
||||
`auto_fin_data_step` 使用财联社网页同源接口的签名和分页方式,从分析时刻开始向前翻页,直到完整覆盖严格的最近 24
|
||||
|
|
@ -82,8 +86,9 @@ daily/YYYY-MM-DD/auto_fin.md
|
|||
Prompt 要求 Agent 只链接实际使用过的历史 Markdown;代码边界则独立保证只保留真实存在、相对
|
||||
workspace 的 Markdown 目标。不存在、绝对路径、越界、带反斜杠和自引用的目标都会降级为可读 alias。
|
||||
|
||||
同日重跑会参考当天已有报告并覆盖为修订结果。最终写入使用原子替换并刷新当天索引;流程不会写入 JSONL、中间 Markdown 或 Agent
|
||||
结构化输出。
|
||||
同日重跑会参考当天已有报告并覆盖为修订结果。最终写入使用原子替换并刷新当天索引,随后通过 `auto_tag_step`
|
||||
更新报告的记忆标签 frontmatter;常规后台文件 watcher 会观察该源文件变化并刷新派生索引。流程不会写入 JSONL、
|
||||
中间 Markdown 或 Agent 结构化输出。
|
||||
|
||||
## 参数
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
[project]
|
||||
name = "reme-auto-fin"
|
||||
version = "0.1.2"
|
||||
version = "0.1.3"
|
||||
description = "Auto Fin example plugin for ReMe."
|
||||
readme = "README.md"
|
||||
license = "Apache-2.0"
|
||||
license-files = ["LICENSE"]
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"reme-ai>=0.4.1.9",
|
||||
"reme-ai>=0.4.1.12",
|
||||
]
|
||||
|
||||
[project.entry-points."reme.plugins"]
|
||||
|
|
|
|||
|
|
@ -104,6 +104,7 @@ class AutoFinMergeStep(AutoFinStep):
|
|||
async def execute(self):
|
||||
"""Research the selected news and persist the validated report."""
|
||||
assert self.context is not None
|
||||
self.context["changes"] = []
|
||||
if self.context.get("auto_fin_skipped"):
|
||||
return self.context.response
|
||||
run_date = date.fromisoformat(str(self._required("auto_fin_date")))
|
||||
|
|
@ -131,6 +132,7 @@ class AutoFinMergeStep(AutoFinStep):
|
|||
markdown = f"# {output.title}\n\n> {output.description}\n\n{output.body}\n\n"
|
||||
markdown += "> 未接入可靠行情数据;本文只提供新闻研究和回顾线索,不提供收益、目标价或买卖建议。\n"
|
||||
report = self._report_path(run_date)
|
||||
change = "modified" if report.is_file() else "added"
|
||||
_write(report, markdown)
|
||||
await refresh_day_index(
|
||||
SimpleNamespace(workspace_path=self.workspace_path),
|
||||
|
|
@ -138,6 +140,7 @@ class AutoFinMergeStep(AutoFinStep):
|
|||
str(self.config_value("daily_dir")),
|
||||
)
|
||||
relative = report.relative_to(self.workspace_path).as_posix()
|
||||
self.context["changes"] = [{"change": change, "path": relative}]
|
||||
self.context["markdown_path"] = relative
|
||||
self.context["auto_fin_digest_path"] = relative
|
||||
self.context.response.answer = output.body
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ application_defaults:
|
|||
- backend: auto_fin_topic_step
|
||||
- backend: auto_fin_merge_step
|
||||
job_tools: [search, read]
|
||||
- backend: auto_tag_step
|
||||
|
||||
auto_fin_cron:
|
||||
backend: cron
|
||||
|
|
|
|||
|
|
@ -219,6 +219,7 @@ async def test_merge_writes_only_final_report_and_validates_historical_links(tmp
|
|||
assert "missing.md" not in report and "outside.md" not in report
|
||||
assert not (tmp_path / "daily" / "2026-08-10" / "auto_fin_news.md").exists()
|
||||
assert not (tmp_path / "resource").exists()
|
||||
assert context["changes"] == [{"change": "added", "path": "daily/2026-08-10/auto_fin.md"}]
|
||||
assert response.metadata["source_paths"] == ["daily/2026-08-01/auto_fin.md"]
|
||||
|
||||
|
||||
|
|
@ -262,8 +263,10 @@ def test_plugin_config_has_default_topics_and_no_intermediate_index_step():
|
|||
"auto_fin_data_step",
|
||||
"auto_fin_topic_step",
|
||||
"auto_fin_merge_step",
|
||||
"auto_tag_step",
|
||||
]
|
||||
assert job["steps"][2]["job_tools"] == ["search", "read"]
|
||||
assert job["steps"][3] == {"backend": "auto_tag_step"}
|
||||
assert jobs["auto_fin_cron"]["cron"] == "0 18 * * *"
|
||||
assert jobs["auto_fin_cron"]["steps"] == job["steps"]
|
||||
assert (
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ their Job configuration under `application_defaults`. Enable the installed plugi
|
|||
### 1. Install ReMe and Daily Paper
|
||||
|
||||
```bash
|
||||
python -m pip install "reme-ai[core]>=0.4.1.9"
|
||||
python -m pip install "reme-ai[core]>=0.4.1.12"
|
||||
reme plugins install reme-daily-paper
|
||||
```
|
||||
|
||||
|
|
@ -51,6 +51,10 @@ To run the Job once without starting a long-lived service:
|
|||
reme start plugins='["daily-paper"]' job=daily_paper topics="Agent memory"
|
||||
```
|
||||
|
||||
Custom application configs must provide `agent_wrapper.default`, a `file_store.default` with an enabled tag index, and
|
||||
the `search`, `read`, `list_tags`, `frontmatter_read`, and `frontmatter_update` Jobs used by Daily Paper and automatic
|
||||
tagging.
|
||||
|
||||
## Pipeline
|
||||
|
||||
```text
|
||||
|
|
@ -64,7 +68,9 @@ download and parse arXiv PDFs, then write three Chinese analyses
|
|||
↓
|
||||
use search + read to connect prior memory and generate a brief
|
||||
↓
|
||||
refresh the daily index and optionally send the brief to DingTalk
|
||||
generate memory tags; the background file watcher refreshes indexes
|
||||
↓
|
||||
optionally send the brief to DingTalk
|
||||
```
|
||||
|
||||
`daily_paper_collect_step` concurrently reads the weekly and monthly rankings for the run date plus the strictly
|
||||
|
|
@ -81,8 +87,10 @@ PDFs and files without a text layer fail explicitly.
|
|||
|
||||
`daily_paper_digest_step` treats those three analyses as the factual source and receives only the read-only
|
||||
`search` and `read` tools for linking earlier memory. Code validates historical wikilinks, appends links to all
|
||||
three source notes, and rebuilds the daily index. The optional `dingtalk_markdown_send_step` sends the final brief when
|
||||
conversation IDs are configured and otherwise skips without side effects.
|
||||
three source notes, and rebuilds the daily index. The workflow then runs `auto_tag_step` to update the memory-tag
|
||||
frontmatter of all three analyses and the final brief. The normal background file watcher observes those source-file
|
||||
changes and refreshes derived indexes before the optional `dingtalk_markdown_send_step` sends the brief. DingTalk
|
||||
delivery skips without side effects when conversation IDs are not configured.
|
||||
|
||||
## Parameters
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ Step backend,并在 `application_defaults` 下提供 Job 配置;通过 `plug
|
|||
### 1. 安装 ReMe 和每日论文插件
|
||||
|
||||
```bash
|
||||
python -m pip install "reme-ai[core]>=0.4.1.9"
|
||||
python -m pip install "reme-ai[core]>=0.4.1.12"
|
||||
reme plugins install reme-daily-paper
|
||||
```
|
||||
|
||||
|
|
@ -47,6 +47,10 @@ curl -s http://127.0.0.1:2333/daily_paper \
|
|||
reme start plugins='["daily-paper"]' job=daily_paper topics="Agent memory"
|
||||
```
|
||||
|
||||
自定义应用配置需要提供 `agent_wrapper.default`、启用 tag index 的 `file_store.default`,以及
|
||||
Daily Paper 和自动标签使用的 `search`、`read`、`list_tags`、`frontmatter_read` 和
|
||||
`frontmatter_update` Jobs。
|
||||
|
||||
## 流程
|
||||
|
||||
```text
|
||||
|
|
@ -60,7 +64,9 @@ RRF 排序后由 Agent 精选三篇
|
|||
↓
|
||||
使用 search + read 关联历史记忆并生成简报
|
||||
↓
|
||||
写入当日索引,并按需发送到钉钉
|
||||
生成记忆标签,由后台文件 watcher 刷新索引
|
||||
↓
|
||||
按需发送到钉钉
|
||||
```
|
||||
|
||||
`daily_paper_collect_step` 并发读取运行日期所在周和所在月的榜单,以及严格前一日的 Daily Papers。候选按 arXiv ID
|
||||
|
|
@ -73,8 +79,9 @@ RRF 排序后由 Agent 精选三篇
|
|||
文本。三篇中文解读按精选顺序写入当天目录;扫描版或没有文本层的 PDF 会明确失败。
|
||||
|
||||
`daily_paper_digest_step` 以本次生成的三篇解读为事实来源,只开放只读的 `search` 和 `read` 来关联较早记忆。
|
||||
代码会校验历史 wikilink、追加三篇源笔记链接,并重建当日索引。可选的 `dingtalk_markdown_send_step` 在配置群会话后
|
||||
发送最终简报;未配置时无副作用跳过。
|
||||
代码会校验历史 wikilink、追加三篇源笔记链接,并重建当日索引。随后 `auto_tag_step` 会更新三篇解读及最终简报的
|
||||
记忆标签 frontmatter,常规后台文件 watcher 会观察这些源文件变化并刷新派生索引,再由可选的
|
||||
`dingtalk_markdown_send_step` 发送最终简报;未配置群会话时无副作用跳过。
|
||||
|
||||
## 参数
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "reme-daily-paper"
|
||||
version = "0.1.2"
|
||||
version = "0.1.3"
|
||||
description = "Daily Paper research and reading-note plugin for ReMe."
|
||||
readme = "README.md"
|
||||
license = "Apache-2.0"
|
||||
|
|
@ -8,7 +8,7 @@ license-files = ["LICENSE"]
|
|||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"pypdf>=5.0.0",
|
||||
"reme-ai>=0.4.1.9",
|
||||
"reme-ai>=0.4.1.12",
|
||||
]
|
||||
|
||||
[project.entry-points."reme.plugins"]
|
||||
|
|
|
|||
|
|
@ -155,8 +155,18 @@ class DailyPaperAnalyzeStep(DailyPaperStep):
|
|||
"pdf_text_truncated": truncated,
|
||||
},
|
||||
)
|
||||
changes = list(self.context.get("changes") or [])
|
||||
if existing_note is not None and existing_note != note_path:
|
||||
existing_rel = existing_note.relative_to(self.workspace_path).as_posix()
|
||||
existing_note.unlink()
|
||||
changes.append({"change": "deleted", "path": existing_rel})
|
||||
changes.append(
|
||||
{
|
||||
"change": "modified" if existing_note == note_path else "added",
|
||||
"path": note_rel,
|
||||
},
|
||||
)
|
||||
self.context["changes"] = changes
|
||||
self.logger.info(
|
||||
f"[{self.name}] paper done arxiv_id={paper.arxiv_id} note_path={note_rel}",
|
||||
)
|
||||
|
|
@ -172,6 +182,7 @@ class DailyPaperAnalyzeStep(DailyPaperStep):
|
|||
|
||||
async def execute(self):
|
||||
assert self.context is not None
|
||||
self.context["changes"] = []
|
||||
if self._skip():
|
||||
self.logger.info(f"[{self.name}] skip existing digest")
|
||||
return self.context.response
|
||||
|
|
|
|||
|
|
@ -108,6 +108,7 @@ class DailyPaperDigestStep(DailyPaperStep):
|
|||
title = normalize_chinese_title(output.title, f"每日论文简报-{day}")
|
||||
existing_rel = str(self._state("existing_digest_path") or "").strip()
|
||||
existing_path = self.workspace_path / existing_rel if existing_rel else None
|
||||
existing_was_file = existing_path is not None and existing_path.is_file()
|
||||
title, digest_path = resolve_unique_note_path(
|
||||
self.workspace_path / daily_dir / day,
|
||||
title,
|
||||
|
|
@ -135,8 +136,13 @@ class DailyPaperDigestStep(DailyPaperStep):
|
|||
"generated_at": utc_now_iso(),
|
||||
},
|
||||
)
|
||||
if existing_path is not None and existing_path != digest_path:
|
||||
changes = list(self.context.get("changes") or [])
|
||||
if existing_was_file and existing_path != digest_path:
|
||||
existing_path.unlink()
|
||||
changes.append({"change": "deleted", "path": existing_rel})
|
||||
digest_change = "modified" if existing_was_file and existing_path == digest_path else "added"
|
||||
changes.append({"change": digest_change, "path": digest_rel})
|
||||
self.context["changes"] = changes
|
||||
self._set_state("digest_path", digest_rel)
|
||||
self.logger.info(f"[{self.name}] digest written path={digest_rel}")
|
||||
self.logger.info(
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ application_defaults:
|
|||
- backend: daily_paper_analyze_step
|
||||
- backend: daily_paper_digest_step
|
||||
job_tools: [search, read]
|
||||
- backend: auto_tag_step
|
||||
- backend: dingtalk_markdown_send_step
|
||||
input_mapping:
|
||||
daily_paper_digest_path: markdown_path
|
||||
|
|
|
|||
|
|
@ -63,7 +63,19 @@ def test_plugin_manifest_declares_complete_runtime_surface():
|
|||
"daily_paper_analyze_step",
|
||||
"daily_paper_digest_step",
|
||||
}
|
||||
assert set(_plugin_config()["jobs"]) == {"daily_paper", "daily_paper_cron"}
|
||||
jobs = _plugin_config()["jobs"]
|
||||
assert set(jobs) == {"daily_paper", "daily_paper_cron"}
|
||||
assert [step["backend"] for step in jobs["daily_paper"]["steps"]] == [
|
||||
"daily_paper_collect_step",
|
||||
"daily_paper_rank_step",
|
||||
"daily_paper_select_step",
|
||||
"daily_paper_analyze_step",
|
||||
"daily_paper_digest_step",
|
||||
"auto_tag_step",
|
||||
"dingtalk_markdown_send_step",
|
||||
]
|
||||
assert jobs["daily_paper"]["steps"][5] == {"backend": "auto_tag_step"}
|
||||
assert jobs["daily_paper_cron"]["steps"] == jobs["daily_paper"]["steps"]
|
||||
|
||||
|
||||
class _QueuedAgentWrapper(BaseAgentWrapper):
|
||||
|
|
@ -898,6 +910,12 @@ async def test_pipeline_filters_strict_yesterday_and_writes_outputs(
|
|||
"Clear evidence",
|
||||
]
|
||||
assert digest.metadata["arxiv_ids"] == ["2607.10001", "2607.10004", "2607.10005"]
|
||||
assert context["changes"] == [
|
||||
{"change": "added", "path": "daily/2026-07-21/记忆代理研究.md"},
|
||||
{"change": "added", "path": "daily/2026-07-21/上下文压缩研究.md"},
|
||||
{"change": "added", "path": "daily/2026-07-21/持续学习研究.md"},
|
||||
{"change": "added", "path": "daily/2026-07-21/今日智能体论文简报.md"},
|
||||
]
|
||||
assert cc_wrapper.calls[0]["kwargs"] == {"output_schema": PaperPickList}
|
||||
assert "用户感兴趣的主题" not in cc_wrapper.calls[0]["inputs"]
|
||||
assert "用户未提供明确的 topic 倾向" in cc_wrapper.calls[0]["inputs"]
|
||||
|
|
@ -984,6 +1002,10 @@ async def test_digest_force_migrates_old_fixed_filename_to_chinese_title(tmp_pat
|
|||
assert new_path.is_file()
|
||||
assert not old_path.exists()
|
||||
assert context["daily_paper_digest_path"] == "daily/2026-07-21/全新论文简报.md"
|
||||
assert context["changes"] == [
|
||||
{"change": "deleted", "path": "daily/2026-07-21/daily-paper-brief.md"},
|
||||
{"change": "added", "path": "daily/2026-07-21/全新论文简报.md"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"""ReMe CLI package."""
|
||||
|
||||
__version__ = "0.4.1.11"
|
||||
__version__ = "0.4.1.12"
|
||||
|
||||
from . import config
|
||||
from . import constants
|
||||
|
|
|
|||
|
|
@ -66,7 +66,12 @@ class LocalFileStore(BaseFileStore):
|
|||
self.embedding_store = self.bind(embedding_store, BaseEmbeddingStore, default_factory=LocalEmbeddingStore)
|
||||
self.keyword_index = self.bind(keyword_index, BaseKeywordIndex, default_factory=BM25Index)
|
||||
self.file_graph = self.bind(file_graph, BaseFileGraph, default_factory=LocalFileGraph)
|
||||
self.tag_index = self.bind(tag_index, BaseTagIndex, default_factory=LocalTagIndex)
|
||||
self.tag_index = self.bind(
|
||||
tag_index,
|
||||
BaseTagIndex,
|
||||
default_factory=LocalTagIndex if tag_index == "default" else None,
|
||||
optional=False,
|
||||
)
|
||||
|
||||
self.encoding = encoding
|
||||
self.store_version = store_version
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from abc import abstractmethod
|
|||
from typing import ClassVar, Literal, TypedDict
|
||||
|
||||
from ..base_component import BaseComponent
|
||||
from ...constants import DEFAULT_MAX_MEMORY_TAG_LENGTH, DEFAULT_MAX_MEMORY_TAGS, DEFAULT_MEMORY_TAG_KEY
|
||||
from ...enumeration import ComponentEnum
|
||||
from ...schema import FileNode
|
||||
|
||||
|
|
@ -28,11 +29,25 @@ class BaseTagIndex(BaseComponent):
|
|||
component_type = ComponentEnum.TAG_INDEX
|
||||
reserved_tag_keys: ClassVar[frozenset[str]] = frozenset()
|
||||
|
||||
def __init__(self, tag_key: object = "memory_tags", **kwargs):
|
||||
def __init__(
|
||||
self,
|
||||
tag_key: object = DEFAULT_MEMORY_TAG_KEY,
|
||||
max_tags_per_file: int = DEFAULT_MAX_MEMORY_TAGS,
|
||||
max_tag_length: int = DEFAULT_MAX_MEMORY_TAG_LENGTH,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self._tag_key = self._validate_tag_key(tag_key)
|
||||
self.max_tags_per_file = self._positive_int("max_tags_per_file", max_tags_per_file)
|
||||
self.max_tag_length = self._positive_int("max_tag_length", max_tag_length)
|
||||
self.is_healthy = True
|
||||
|
||||
@staticmethod
|
||||
def _positive_int(name: str, value: object) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
|
||||
raise ValueError(f"{name} must be a positive integer")
|
||||
return value
|
||||
|
||||
@property
|
||||
def tag_key(self) -> str:
|
||||
"""Frontmatter field from which this index derives tags."""
|
||||
|
|
|
|||
|
|
@ -20,16 +20,8 @@ class LocalTagIndex(BaseTagIndex):
|
|||
"status",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tag_key: object = "memory_tags",
|
||||
max_tags_per_file: int = 3,
|
||||
max_tag_length: int = 64,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(tag_key=tag_key, **kwargs)
|
||||
self.max_tags_per_file = self._positive_int("max_tags_per_file", max_tags_per_file)
|
||||
self.max_tag_length = self._positive_int("max_tag_length", max_tag_length)
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.path_to_tags: dict[str, tuple[str, ...]] = {}
|
||||
self.tag_to_paths: dict[str, set[str]] = {}
|
||||
self._maintenance_lock = asyncio.Lock()
|
||||
|
|
@ -38,12 +30,6 @@ class LocalTagIndex(BaseTagIndex):
|
|||
def n_files(self) -> int:
|
||||
return len(self.path_to_tags)
|
||||
|
||||
@staticmethod
|
||||
def _positive_int(name: str, value: object) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
|
||||
raise ValueError(f"{name} must be a positive integer")
|
||||
return value
|
||||
|
||||
def _normalize_tags(self, value: object, *, limit: int | None) -> list[str]:
|
||||
"""Normalize a strict tag list, optionally limiting the result count."""
|
||||
if not isinstance(value, list):
|
||||
|
|
|
|||
|
|
@ -189,7 +189,6 @@ jobs:
|
|||
steps:
|
||||
- backend: auto_memory_step
|
||||
- backend: auto_tag_step
|
||||
max_tags_per_file: 3
|
||||
|
||||
auto_memory_cc:
|
||||
backend: base
|
||||
|
|
@ -208,7 +207,6 @@ jobs:
|
|||
steps:
|
||||
- backend: auto_memory_cc_step
|
||||
- backend: auto_tag_step
|
||||
max_tags_per_file: 3
|
||||
|
||||
auto_resource:
|
||||
backend: base
|
||||
|
|
|
|||
|
|
@ -18,3 +18,8 @@ DEFAULT_MAX_IMAGE_BYTES = 5 * 1024 * 1024
|
|||
# Background content-processing jobs skip files above this size. File watchers
|
||||
# and catalogs still track them so deletes and later size reductions are seen.
|
||||
DEFAULT_MAX_FILE_BYTES = 20 * 1024 * 1024
|
||||
|
||||
# Memory-tag generation and indexing defaults.
|
||||
DEFAULT_MEMORY_TAG_KEY = "memory_tags"
|
||||
DEFAULT_MAX_MEMORY_TAGS = 3
|
||||
DEFAULT_MAX_MEMORY_TAG_LENGTH = 64
|
||||
|
|
|
|||
|
|
@ -71,12 +71,18 @@ class Ref:
|
|||
obj.__dict__.pop(self._cache_attr, None)
|
||||
|
||||
def _resolve(self, obj: "BaseStep"):
|
||||
for source in (obj.kwargs, obj.context or {}):
|
||||
sources = (obj.kwargs, obj.context or {})
|
||||
for source in sources:
|
||||
value = source.get(self.key)
|
||||
if isinstance(value, self.base_cls):
|
||||
return value
|
||||
|
||||
name = obj.kwargs.get(self.key, "default")
|
||||
name = "default"
|
||||
for source in sources:
|
||||
value = source.get(self.key)
|
||||
if isinstance(value, str):
|
||||
name = value
|
||||
break
|
||||
if obj.app_context is None:
|
||||
if self.optional:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -1,20 +1,17 @@
|
|||
"""Generate entity-oriented memory tags for added or modified Markdown files."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
import frontmatter
|
||||
|
||||
from ._evolve import agent_reply_result_text
|
||||
from ..base_step import BaseStep
|
||||
from ..file_io import parse_daily_date, refresh_day_index
|
||||
from ..file_io._path import display_path, resolve_path
|
||||
from ..index import normalize_posix_path
|
||||
from ...components import R
|
||||
from ...constants import DEFAULT_MAX_MEMORY_TAG_LENGTH, DEFAULT_MAX_MEMORY_TAGS
|
||||
|
||||
_DEFAULT_MAX_MEMORY_TAGS = 3
|
||||
_DEFAULT_MAX_MEMORY_TAG_LENGTH = 64
|
||||
_SUPPORTED_CHANGES = {"added", "modified"}
|
||||
|
||||
|
||||
|
|
@ -24,21 +21,13 @@ class _TagTarget:
|
|||
path: str
|
||||
|
||||
|
||||
def _positive_int(value: object, *, name: str) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
|
||||
raise ValueError(f"{name} must be a positive integer")
|
||||
return value
|
||||
|
||||
|
||||
def normalize_memory_tags(
|
||||
value: object,
|
||||
*,
|
||||
max_tags_per_file: int = _DEFAULT_MAX_MEMORY_TAGS,
|
||||
max_tag_length: int = _DEFAULT_MAX_MEMORY_TAG_LENGTH,
|
||||
max_tags_per_file: int = DEFAULT_MAX_MEMORY_TAGS,
|
||||
max_tag_length: int = DEFAULT_MAX_MEMORY_TAG_LENGTH,
|
||||
) -> list[str]:
|
||||
"""Normalize human-readable entity labels for frontmatter storage."""
|
||||
max_tags_per_file = _positive_int(max_tags_per_file, name="max_tags_per_file")
|
||||
max_tag_length = _positive_int(max_tag_length, name="max_tag_length")
|
||||
"""Normalize source tags while preserving canonical display casing."""
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
|
||||
|
|
@ -64,22 +53,10 @@ def normalize_memory_tags(
|
|||
class AutoTagStep(BaseStep):
|
||||
"""Update memory tags for Markdown files described by the common ``changes`` contract."""
|
||||
|
||||
def __init__(self, max_tags_per_file: int = _DEFAULT_MAX_MEMORY_TAGS, **kwargs):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.max_tags_per_file = _positive_int(max_tags_per_file, name="max_tags_per_file")
|
||||
self.tools = ["read", "list_tags", "frontmatter_read", "frontmatter_update"]
|
||||
|
||||
@staticmethod
|
||||
def _index_limit(tag_index, name: str, fallback: int | None) -> int | None:
|
||||
"""Read one positive integer index limit, falling back when unavailable or invalid."""
|
||||
try:
|
||||
value = getattr(tag_index, name)
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
return fallback
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
|
||||
return fallback
|
||||
return value
|
||||
|
||||
def _targets(self) -> tuple[list[_TagTarget], list[dict[str, str]]]:
|
||||
"""Validate, normalize, and de-duplicate added/modified Markdown changes."""
|
||||
assert self.context is not None
|
||||
|
|
@ -87,7 +64,7 @@ class AutoTagStep(BaseStep):
|
|||
if not isinstance(raw_changes, list):
|
||||
raise ValueError("AutoTagStep requires changes: list[dict]")
|
||||
|
||||
workspace = Path(self.file_store.workspace_path or ".").resolve()
|
||||
workspace = self.workspace_path.resolve()
|
||||
targets: dict[str, _TagTarget] = {}
|
||||
ignored: list[dict[str, str]] = []
|
||||
for item in raw_changes:
|
||||
|
|
@ -98,7 +75,12 @@ class AutoTagStep(BaseStep):
|
|||
change = str(item.get("change") or "").strip().lower()
|
||||
raw_path = str(item.get("path") or "").strip()
|
||||
if change not in _SUPPORTED_CHANGES:
|
||||
ignored.append({"path": raw_path, "reason": f"unsupported change: {change or 'missing'}"})
|
||||
ignored.append(
|
||||
{
|
||||
"path": raw_path,
|
||||
"reason": f"unsupported change: {change or 'missing'}",
|
||||
},
|
||||
)
|
||||
continue
|
||||
|
||||
target, error = resolve_path(workspace, raw_path)
|
||||
|
|
@ -119,48 +101,44 @@ class AutoTagStep(BaseStep):
|
|||
targets[path] = _TagTarget(change=normalized_change, path=path)
|
||||
return list(targets.values()), ignored
|
||||
|
||||
async def _process_target(self, target: _TagTarget, tag_key: str, max_tag_length: int) -> str:
|
||||
async def _process_target(self, target: _TagTarget) -> str:
|
||||
file_store = self.file_store
|
||||
tag_index = file_store.require_tag_index()
|
||||
tag_key = tag_index.tag_key
|
||||
tool_context = {
|
||||
"file_store": file_store.name,
|
||||
"_allowed_paths": [target.path],
|
||||
"_allowed_frontmatter_keys": [tag_key],
|
||||
}
|
||||
result = await self.agent_wrapper.reply(
|
||||
self.prompt_format("user_message", path=target.path, change=target.change, tag_key=tag_key),
|
||||
system_prompt=self.prompt_format(
|
||||
"system_prompt",
|
||||
tag_key=tag_key,
|
||||
max_tags_per_file=self.max_tags_per_file,
|
||||
max_tags_per_file=tag_index.max_tags_per_file,
|
||||
),
|
||||
job_tools=self.tools,
|
||||
injected_job_kwargs={
|
||||
"_allowed_paths": [target.path],
|
||||
"_allowed_frontmatter_keys": [tag_key],
|
||||
},
|
||||
injected_job_kwargs=tool_context,
|
||||
)
|
||||
|
||||
path = Path(self.file_store.workspace_path or ".") / target.path
|
||||
path = self.workspace_path / target.path
|
||||
metadata = dict(frontmatter.loads(path.read_text(encoding="utf-8")).metadata or {})
|
||||
normalized = normalize_memory_tags(
|
||||
metadata.get(tag_key),
|
||||
max_tags_per_file=self.max_tags_per_file,
|
||||
max_tag_length=max_tag_length,
|
||||
max_tags_per_file=tag_index.max_tags_per_file,
|
||||
max_tag_length=tag_index.max_tag_length,
|
||||
)
|
||||
if metadata.get(tag_key) != normalized:
|
||||
response = await self.run_job(
|
||||
"frontmatter_update",
|
||||
path=target.path,
|
||||
metadata={tag_key: normalized},
|
||||
_allowed_paths=[target.path],
|
||||
_allowed_frontmatter_keys=[tag_key],
|
||||
**tool_context,
|
||||
)
|
||||
if not response.success:
|
||||
raise RuntimeError(str(response.answer))
|
||||
return agent_reply_result_text(result)
|
||||
|
||||
def _daily_date(self, path: str) -> str | None:
|
||||
daily_dir = normalize_posix_path(str(self.config_value("daily_dir"))).strip("/")
|
||||
prefix = f"{daily_dir}/"
|
||||
if not path.startswith(prefix):
|
||||
return None
|
||||
parts = path[len(prefix) :].split("/")
|
||||
return parse_daily_date(parts[0]) if len(parts) == 2 else None
|
||||
|
||||
async def execute(self):
|
||||
assert self.context is not None
|
||||
initial_success = self.context.response.success
|
||||
|
|
@ -173,54 +151,34 @@ class AutoTagStep(BaseStep):
|
|||
return self.context.response
|
||||
|
||||
results: list[dict] = []
|
||||
indexes: list[dict] = []
|
||||
if targets and not self.file_store.tag_index_enabled:
|
||||
self.context.response.success = False
|
||||
if initial_success:
|
||||
self.context.response.answer = "Error: tag index is not configured"
|
||||
return self.context.response
|
||||
|
||||
tag_index = self.file_store.require_tag_index() if targets else None
|
||||
if tag_index is not None and not tag_index.is_healthy:
|
||||
self.context.response.success = False
|
||||
if initial_success:
|
||||
self.context.response.answer = "Error: tag index unavailable"
|
||||
return self.context.response
|
||||
max_tag_length = self._index_limit(tag_index, "max_tag_length", _DEFAULT_MAX_MEMORY_TAG_LENGTH)
|
||||
index_max_tags = self._index_limit(tag_index, "max_tags_per_file", None)
|
||||
if index_max_tags is not None and self.max_tags_per_file > index_max_tags:
|
||||
self.context.response.success = False
|
||||
if initial_success:
|
||||
self.context.response.answer = (
|
||||
f"Error: auto_tag max_tags_per_file ({self.max_tags_per_file}) exceeds "
|
||||
f"tag index limit ({index_max_tags})"
|
||||
)
|
||||
return self.context.response
|
||||
|
||||
dates: set[str] = set()
|
||||
for target in targets:
|
||||
if day := self._daily_date(target.path):
|
||||
dates.add(day)
|
||||
try:
|
||||
summary = await self._process_target(target, tag_index.tag_key, max_tag_length)
|
||||
summary = await self._process_target(target)
|
||||
results.append(
|
||||
{"change": target.change, "path": target.path, "success": True, "summary": summary},
|
||||
{
|
||||
"change": target.change,
|
||||
"path": target.path,
|
||||
"success": True,
|
||||
"summary": summary,
|
||||
},
|
||||
)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
results.append(
|
||||
{"change": target.change, "path": target.path, "success": False, "error": str(exc)},
|
||||
{
|
||||
"change": target.change,
|
||||
"path": target.path,
|
||||
"success": False,
|
||||
"error": str(exc),
|
||||
},
|
||||
)
|
||||
self.logger.warning(f"[{self.name}] failed path={target.path}: {exc}")
|
||||
|
||||
for day in sorted(dates):
|
||||
indexes.append(await refresh_day_index(self.file_store, day, self.config_value("daily_dir")))
|
||||
|
||||
failed = sum(not item["success"] for item in results)
|
||||
succeeded = len(results) - failed
|
||||
self.context.response.success = initial_success and failed == 0
|
||||
if initial_success and failed:
|
||||
self.context.response.success = initial_success
|
||||
if not initial_answer and failed:
|
||||
self.context.response.answer = f"Tagged {succeeded} file(s); {failed} failed"
|
||||
elif initial_success and not initial_answer and succeeded:
|
||||
elif not initial_answer and succeeded:
|
||||
self.context.response.answer = f"Tagged {succeeded} file(s)"
|
||||
else:
|
||||
self.context.response.answer = initial_answer
|
||||
|
|
@ -230,6 +188,5 @@ class AutoTagStep(BaseStep):
|
|||
"failed": failed,
|
||||
"ignored": ignored,
|
||||
"results": results,
|
||||
"indexes": indexes,
|
||||
}
|
||||
return self.context.response
|
||||
|
|
|
|||
|
|
@ -25,8 +25,14 @@ class _ForwardToLoggerHandler(logging.Handler):
|
|||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
target = logging.getLogger(self.target_name)
|
||||
if target.isEnabledFor(record.levelno):
|
||||
target.handle(record)
|
||||
# Python 3.13 treats both ``isEnabledFor`` and ``handle`` as disabled
|
||||
# during a nested logger call. Apply their stable public checks here,
|
||||
# then dispatch directly so forwarded ReMe records are not dropped.
|
||||
if target.disabled or target.manager.disable >= record.levelno or record.levelno < target.getEffectiveLevel():
|
||||
return
|
||||
filtered = target.filter(record)
|
||||
if filtered:
|
||||
target.callHandlers(filtered if isinstance(filtered, logging.LogRecord) else record)
|
||||
|
||||
|
||||
class _QwenPawStdlibFormatter(logging.Formatter):
|
||||
|
|
|
|||
287
tests/integration/test_auto_tag.py
Normal file
287
tests/integration/test_auto_tag.py
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
"""Real-LLM integration tests for ``auto_tag_step``.
|
||||
|
||||
The fixtures mimic the final Markdown written by the Auto Fin and Daily Paper
|
||||
plugins, then invoke AutoTagStep through a normal application Job. The tests
|
||||
load the repository ``.env`` and therefore call the configured real LLM.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import frontmatter
|
||||
|
||||
INTEGRATION_DIR = Path(__file__).resolve().parent
|
||||
REPOSITORY = INTEGRATION_DIR.parents[1]
|
||||
sys.path.insert(0, str(REPOSITORY))
|
||||
sys.path.insert(0, str(INTEGRATION_DIR))
|
||||
|
||||
# pylint: disable=wrong-import-position
|
||||
from _workspace_fixture import workspace_env # noqa: E402
|
||||
|
||||
from reme.utils import load_env # noqa: E402
|
||||
|
||||
AUTO_TAG_JOB = {
|
||||
"integration_auto_tag": {
|
||||
"backend": "base",
|
||||
"enable_serve": False,
|
||||
"steps": [{"backend": "auto_tag_step"}],
|
||||
},
|
||||
}
|
||||
|
||||
AUTO_FIN_REPORT = """\
|
||||
---
|
||||
name: auto_fin
|
||||
description: 宁德时代海外电池业务与产能进展的盘后研究
|
||||
kind: auto-fin-brief
|
||||
---
|
||||
|
||||
# 宁德时代:欧洲电池业务进入产能兑现期
|
||||
|
||||
> 本文围绕宁德时代这一家公司,复盘其欧洲电池工厂的量产进展与客户交付节奏。
|
||||
|
||||
## 核心变化
|
||||
|
||||
宁德时代确认匈牙利工厂首条电芯产线开始试生产。管理层称,后续爬坡速度仍取决于良率、当地供应链和
|
||||
客户认证。该进展可能缩短欧洲客户的交付半径,但资本开支和产能利用率仍是主要风险。
|
||||
|
||||
## 观察框架
|
||||
|
||||
后续应继续跟踪宁德时代的海外产能利用率、单位成本与订单兑现情况。本文只提供新闻研究和回顾线索,
|
||||
不提供收益、目标价或买卖建议。
|
||||
"""
|
||||
|
||||
DAILY_PAPER_NOTES = {
|
||||
"daily/2026-09-10/openai-agent-eval.md": """\
|
||||
---
|
||||
name: openai-agent-eval
|
||||
description: OpenAI 智能体可靠性评测论文解读
|
||||
kind: daily-paper-analysis
|
||||
arxiv_id: "2609.10001"
|
||||
---
|
||||
|
||||
# OpenAI 智能体可靠性评测
|
||||
|
||||
这篇论文以 OpenAI 为唯一机构研究对象,分析其智能体在长任务中的失败恢复机制。实验比较了重试预算、
|
||||
工具错误和上下文压缩对完成率的影响,并讨论 OpenAI 对可靠性评测的设计选择。
|
||||
""",
|
||||
"daily/2026-09-10/anthropic-context.md": """\
|
||||
---
|
||||
name: anthropic-context
|
||||
description: Anthropic 长上下文研究论文解读
|
||||
kind: daily-paper-analysis
|
||||
arxiv_id: "2609.10002"
|
||||
---
|
||||
|
||||
# Anthropic 长上下文研究
|
||||
|
||||
论文只研究 Anthropic 的长上下文模型。作者测试信息位于不同位置时的召回差异,并分析 Anthropic 模型的
|
||||
注意力退化现象;上下文压缩只是实验方法,不是本文要标记的现实实体。
|
||||
""",
|
||||
"daily/2026-09-10/nvidia-blackwell.md": """\
|
||||
---
|
||||
name: nvidia-blackwell
|
||||
description: NVIDIA Blackwell 训练系统论文解读
|
||||
kind: daily-paper-analysis
|
||||
arxiv_id: "2609.10003"
|
||||
---
|
||||
|
||||
# NVIDIA Blackwell 训练系统
|
||||
|
||||
本文的机构研究对象是 NVIDIA。论文评估 Blackwell 集群的大模型训练吞吐、故障恢复和互连扩展效率,
|
||||
并给出 NVIDIA 在超大规模训练系统上的工程取舍。
|
||||
""",
|
||||
}
|
||||
|
||||
DAILY_PAPER_DIGEST = """\
|
||||
---
|
||||
name: 每日论文简报-2026-09-10
|
||||
description: OpenAI、Anthropic 与 NVIDIA 三项研究的每日论文简报
|
||||
kind: daily-paper-brief
|
||||
---
|
||||
|
||||
# 每日论文简报:智能体、长上下文与训练系统
|
||||
|
||||
今天的三篇论文分别围绕三个相互独立的机构实体展开:OpenAI 的智能体可靠性评测、Anthropic 的长上下文
|
||||
研究,以及 NVIDIA 的 Blackwell 训练系统。这三家机构都是本期简报的并列核心,而非顺带提及。
|
||||
|
||||
## 来源
|
||||
|
||||
- [[daily/2026-09-10/openai-agent-eval.md]]
|
||||
- [[daily/2026-09-10/anthropic-context.md]]
|
||||
- [[daily/2026-09-10/nvidia-blackwell.md]]
|
||||
"""
|
||||
|
||||
GENERIC_TAGS = {
|
||||
"ai",
|
||||
"人工智能",
|
||||
"金融",
|
||||
"股票",
|
||||
"论文",
|
||||
"研究",
|
||||
"智能体",
|
||||
"长上下文",
|
||||
"训练系统",
|
||||
"电池",
|
||||
"半导体",
|
||||
}
|
||||
|
||||
|
||||
def _write(workspace: Path, relative: str, content: str) -> Path:
|
||||
target = workspace / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(content, encoding="utf-8")
|
||||
return target
|
||||
|
||||
|
||||
def _post(path: Path) -> frontmatter.Post:
|
||||
return frontmatter.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _tags(path: Path) -> list[str]:
|
||||
value = _post(path).metadata.get("memory_tags")
|
||||
assert isinstance(value, list), f"memory_tags was not written as a list: {value!r}"
|
||||
assert all(isinstance(item, str) and item.strip() for item in value)
|
||||
assert len(value) <= 3
|
||||
assert not ({item.casefold() for item in value} & {item.casefold() for item in GENERIC_TAGS})
|
||||
return value
|
||||
|
||||
|
||||
def _assert_tag(tags: list[str], expected: str) -> None:
|
||||
assert expected.casefold() in {tag.casefold() for tag in tags}, f"expected entity {expected!r}, got {tags!r}"
|
||||
|
||||
|
||||
def _assert_any_tag(tags: list[str], expected: set[str]) -> None:
|
||||
actual = {tag.casefold() for tag in tags}
|
||||
accepted = {tag.casefold() for tag in expected}
|
||||
assert actual & accepted, f"expected one of {sorted(expected)!r}, got {tags!r}"
|
||||
|
||||
|
||||
async def _run_tag_job(env, changes: list[dict[str, str]]):
|
||||
app = await env.make_app(jobs=AUTO_TAG_JOB)
|
||||
response = await app.run_job("integration_auto_tag", changes=changes)
|
||||
assert response.success is True, f"auto-tag failed: {response.answer!r}; metadata={response.metadata!r}"
|
||||
auto_tag = response.metadata["auto_tag"]
|
||||
assert auto_tag["processed"] == len(changes)
|
||||
assert auto_tag["succeeded"] == len(changes)
|
||||
assert auto_tag["failed"] == 0
|
||||
assert auto_tag["ignored"] == []
|
||||
return app, response
|
||||
|
||||
|
||||
def test_auto_tag_auto_fin_report_uses_company_entity():
|
||||
"""An Auto Fin report should be tagged with its company, not broad finance topics."""
|
||||
|
||||
async def run():
|
||||
load_env(REPOSITORY / ".env")
|
||||
with workspace_env(load_env_file=False) as env:
|
||||
relative = "daily/2026-09-10/auto_fin.md"
|
||||
path = _write(env.workspace_dir, relative, AUTO_FIN_REPORT)
|
||||
before = _post(path)
|
||||
try:
|
||||
_app, _response = await _run_tag_job(env, [{"change": "added", "path": relative}])
|
||||
tags = _tags(path)
|
||||
_assert_any_tag(tags, {"宁德时代", "CATL"})
|
||||
assert len(tags) == 1, f"single-entity report received extra tags: {tags!r}"
|
||||
after = _post(path)
|
||||
assert after.content == before.content
|
||||
assert {key: value for key, value in after.metadata.items() if key != "memory_tags"} == before.metadata
|
||||
finally:
|
||||
await env.close_all()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_auto_tag_daily_paper_batch_tags_analyses_and_digest():
|
||||
"""The three analyses get one entity each and the digest gets all three."""
|
||||
|
||||
async def run():
|
||||
load_env(REPOSITORY / ".env")
|
||||
with workspace_env(load_env_file=False) as env:
|
||||
paths = {
|
||||
relative: _write(env.workspace_dir, relative, content)
|
||||
for relative, content in DAILY_PAPER_NOTES.items()
|
||||
}
|
||||
digest_rel = "daily/2026-09-10/daily-paper-brief.md"
|
||||
paths[digest_rel] = _write(env.workspace_dir, digest_rel, DAILY_PAPER_DIGEST)
|
||||
before = {relative: _post(path) for relative, path in paths.items()}
|
||||
changes = [{"change": "added", "path": relative} for relative in paths]
|
||||
try:
|
||||
_app, response = await _run_tag_job(env, changes)
|
||||
expected = {
|
||||
"daily/2026-09-10/openai-agent-eval.md": "OpenAI",
|
||||
"daily/2026-09-10/anthropic-context.md": "Anthropic",
|
||||
"daily/2026-09-10/nvidia-blackwell.md": "NVIDIA",
|
||||
}
|
||||
for relative, entity in expected.items():
|
||||
tags = _tags(paths[relative])
|
||||
assert len(tags) == 1, f"single-entity analysis received extra tags: {relative} -> {tags!r}"
|
||||
_assert_tag(tags, entity)
|
||||
|
||||
digest_tags = _tags(paths[digest_rel])
|
||||
assert {tag.casefold() for tag in digest_tags} == {
|
||||
"openai",
|
||||
"anthropic",
|
||||
"nvidia",
|
||||
}
|
||||
for relative, path in paths.items():
|
||||
after = _post(path)
|
||||
assert after.content == before[relative].content
|
||||
assert {key: value for key, value in after.metadata.items() if key != "memory_tags"} == before[
|
||||
relative
|
||||
].metadata
|
||||
auto_tag = response.metadata["auto_tag"]
|
||||
assert [item["path"] for item in auto_tag["results"]] == list(paths)
|
||||
finally:
|
||||
await env.close_all()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_auto_tag_modified_note_reuses_existing_canonical_tag():
|
||||
"""A modified note should reuse an existing workspace spelling for the same entity."""
|
||||
|
||||
async def run():
|
||||
load_env(REPOSITORY / ".env")
|
||||
with workspace_env(load_env_file=False) as env:
|
||||
_write(
|
||||
env.workspace_dir,
|
||||
"daily/2026-09-09/openai-history.md",
|
||||
"---\nname: openai-history\nmemory_tags: [OpenAI]\n---\n# OpenAI 历史记录\n",
|
||||
)
|
||||
relative = "daily/2026-09-10/openai-update.md"
|
||||
target = _write(
|
||||
env.workspace_dir,
|
||||
relative,
|
||||
"""\
|
||||
---
|
||||
name: openai-update
|
||||
description: OpenAI, Inc. 产品更新记录
|
||||
memory_tags: [大模型]
|
||||
status: reviewed
|
||||
---
|
||||
|
||||
# OpenAI, Inc. 产品更新
|
||||
|
||||
这份记忆只围绕 OpenAI 公司,记录其产品发布节奏。文中的“大模型”是技术类别,不是现实实体标签。
|
||||
""",
|
||||
)
|
||||
body_before = _post(target).content
|
||||
try:
|
||||
_app, response = await _run_tag_job(env, [{"change": "modified", "path": relative}])
|
||||
tags = _tags(target)
|
||||
assert [tag.casefold() for tag in tags] == ["openai"]
|
||||
post = _post(target)
|
||||
assert post.content == body_before
|
||||
assert post.metadata["status"] == "reviewed"
|
||||
assert response.metadata["auto_tag"]["results"][0]["change"] == "modified"
|
||||
finally:
|
||||
await env.close_all()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_auto_tag_auto_fin_report_uses_company_entity()
|
||||
test_auto_tag_daily_paper_batch_tags_analyses_and_digest()
|
||||
test_auto_tag_modified_note_reuses_existing_canonical_tag()
|
||||
|
|
@ -12,7 +12,7 @@ from reme.components.file_store import LocalFileStore
|
|||
from reme.components.runtime_context import RuntimeContext
|
||||
from reme.components.tag_index import LocalTagIndex
|
||||
from reme.schema import Response
|
||||
from reme.steps.evolve.auto_tag import AutoTagStep, normalize_memory_tags
|
||||
from reme.steps.evolve.auto_tag import AutoTagStep
|
||||
|
||||
|
||||
class _TaggingWrapper(BaseAgentWrapper):
|
||||
|
|
@ -49,20 +49,25 @@ def _write_note(path: Path) -> None:
|
|||
path.write_text("---\nname: note\ndescription: useful note\n---\nbody\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _file_store(*, name: str = "default", **tag_index_kwargs) -> LocalFileStore:
|
||||
store = LocalFileStore(name=name, embedding_store="", tag_index="")
|
||||
store.tag_index = LocalTagIndex(**tag_index_kwargs)
|
||||
return store
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_tag_handles_noop_and_rejects_invalid_preconditions(tmp_path, monkeypatch):
|
||||
async def test_auto_tag_handles_noop_invalid_changes_and_no_tag_index(tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
wrapper = _TaggingWrapper(tmp_path)
|
||||
unindexed_store = LocalFileStore(name="store", embedding_store="", tag_index="")
|
||||
|
||||
context = RuntimeContext(changes=[])
|
||||
context.response.answer = "Skipped: no messages"
|
||||
response = await AutoTagStep(file_store=unindexed_store, agent_wrapper=wrapper)(context)
|
||||
response = await AutoTagStep(agent_wrapper=wrapper)(context)
|
||||
assert response.success is True
|
||||
assert response.answer == "Skipped: no messages"
|
||||
assert response.metadata["auto_tag"]["processed"] == 0
|
||||
|
||||
response = await AutoTagStep(file_store=unindexed_store, agent_wrapper=wrapper)(
|
||||
response = await AutoTagStep(agent_wrapper=wrapper)(
|
||||
RuntimeContext(changes="daily/note.md"),
|
||||
)
|
||||
assert response.success is False
|
||||
|
|
@ -70,36 +75,16 @@ async def test_auto_tag_handles_noop_and_rejects_invalid_preconditions(tmp_path,
|
|||
|
||||
note = tmp_path / "daily/2026-09-09/note.md"
|
||||
_write_note(note)
|
||||
before = note.read_bytes()
|
||||
change = RuntimeContext(changes=[{"change": "added", "path": "daily/2026-09-09/note.md"}])
|
||||
response = await AutoTagStep(file_store=unindexed_store, agent_wrapper=wrapper)(change)
|
||||
|
||||
assert response.success is False
|
||||
assert response.answer == "Error: tag index is not configured"
|
||||
assert not wrapper.calls
|
||||
assert note.read_bytes() == before
|
||||
|
||||
indexed_store = LocalFileStore(name="store", embedding_store="", tag_index="")
|
||||
indexed_store.tag_index = LocalTagIndex(max_tags_per_file=2)
|
||||
response = await AutoTagStep(
|
||||
file_store=indexed_store,
|
||||
agent_wrapper=wrapper,
|
||||
max_tags_per_file=3,
|
||||
)(RuntimeContext(changes=[{"change": "added", "path": "daily/2026-09-09/note.md"}]))
|
||||
assert response.success is False
|
||||
assert response.answer == "Error: auto_tag max_tags_per_file (3) exceeds tag index limit (2)"
|
||||
assert not wrapper.calls
|
||||
assert note.read_bytes() == before
|
||||
|
||||
indexed_store.tag_index = LocalTagIndex()
|
||||
indexed_store.tag_index.set_healthy(False)
|
||||
response = await AutoTagStep(file_store=indexed_store, agent_wrapper=wrapper)(
|
||||
RuntimeContext(changes=[{"change": "added", "path": "daily/2026-09-09/note.md"}]),
|
||||
response = await AutoTagStep(agent_wrapper=wrapper, file_store=LocalFileStore(embedding_store="", tag_index=""))(
|
||||
change,
|
||||
)
|
||||
assert response.success is False
|
||||
assert response.answer == "Error: tag index unavailable"
|
||||
|
||||
assert response.success is True
|
||||
assert response.answer == "Tagged 0 file(s); 1 failed"
|
||||
assert not wrapper.calls
|
||||
assert note.read_bytes() == before
|
||||
assert "memory_tags" not in frontmatter.loads(note.read_text(encoding="utf-8")).metadata
|
||||
assert response.metadata["auto_tag"]["results"][0]["error"] == "tag index is not configured"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -112,10 +97,8 @@ async def test_auto_tag_filters_paths_and_continues_after_one_file_fails(tmp_pat
|
|||
(tmp_path / "daily/2026-09-09/notes").mkdir()
|
||||
(tmp_path / "daily/2026-09-09/plain.txt").write_text("text", encoding="utf-8")
|
||||
|
||||
store = LocalFileStore(name="store", embedding_store="", tag_index="")
|
||||
store.tag_index = LocalTagIndex()
|
||||
wrapper = _TaggingWrapper(tmp_path, fail_name="failed.md")
|
||||
step = AutoTagStep(file_store=store, agent_wrapper=wrapper)
|
||||
step = AutoTagStep(agent_wrapper=wrapper, file_store=_file_store())
|
||||
context = RuntimeContext(
|
||||
changes=[
|
||||
{"change": "modified", "path": "daily/2026-09-09/failed.md"},
|
||||
|
|
@ -126,16 +109,20 @@ async def test_auto_tag_filters_paths_and_continues_after_one_file_fails(tmp_pat
|
|||
{"change": "deleted", "path": "daily/2026-09-09/deleted.md"},
|
||||
],
|
||||
)
|
||||
context.response.answer = "Generated report"
|
||||
|
||||
response = await step(context)
|
||||
|
||||
assert response.success is False
|
||||
assert response.success is True
|
||||
assert response.answer == "Generated report"
|
||||
assert [call[1]["injected_job_kwargs"] for call in wrapper.calls] == [
|
||||
{
|
||||
"file_store": "default",
|
||||
"_allowed_paths": ["daily/2026-09-09/failed.md"],
|
||||
"_allowed_frontmatter_keys": ["memory_tags"],
|
||||
},
|
||||
{
|
||||
"file_store": "default",
|
||||
"_allowed_paths": ["daily/2026-09-09/first.md"],
|
||||
"_allowed_frontmatter_keys": ["memory_tags"],
|
||||
},
|
||||
|
|
@ -168,7 +155,6 @@ async def test_auto_tag_filters_paths_and_continues_after_one_file_fails(tmp_pat
|
|||
"summary": "tagged daily/2026-09-09/first.md",
|
||||
},
|
||||
]
|
||||
assert "memory_tags: ['宁德时代', '黄金']" in (tmp_path / "daily/2026-09-09.md").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -176,18 +162,18 @@ async def test_auto_tag_uses_configured_key_and_normalizes_agent_output(tmp_path
|
|||
monkeypatch.chdir(tmp_path)
|
||||
note = tmp_path / "memory/note.md"
|
||||
_write_note(note)
|
||||
store = LocalFileStore(name="store", embedding_store="", tag_index="")
|
||||
store.tag_index = LocalTagIndex(tag_key="keywords", max_tag_length=8)
|
||||
wrapper = _TaggingWrapper(
|
||||
tmp_path,
|
||||
tag_key="keywords",
|
||||
tags=["OpenAI", "openai", "Sam Altman", "++", 100, "宁德时代", "黄金"],
|
||||
)
|
||||
step = AutoTagStep(file_store=store, agent_wrapper=wrapper, max_tags_per_file=2)
|
||||
store = _file_store(name="archive", tag_key="keywords", max_tags_per_file=2, max_tag_length=8)
|
||||
step = AutoTagStep(agent_wrapper=wrapper, file_store=store)
|
||||
|
||||
async def update_frontmatter(name, /, **kwargs):
|
||||
assert name == "frontmatter_update"
|
||||
assert kwargs["_allowed_frontmatter_keys"] == ["keywords"]
|
||||
assert kwargs["file_store"] == "archive"
|
||||
post = frontmatter.loads(note.read_text(encoding="utf-8"))
|
||||
post.metadata.update(kwargs["metadata"])
|
||||
note.write_text(frontmatter.dumps(post), encoding="utf-8")
|
||||
|
|
@ -205,8 +191,4 @@ async def test_auto_tag_uses_configured_key_and_normalizes_agent_output(tmp_path
|
|||
"OpenAI",
|
||||
"宁德时代",
|
||||
]
|
||||
assert normalize_memory_tags(
|
||||
["one", "two", "three"],
|
||||
max_tags_per_file=2,
|
||||
max_tag_length=3,
|
||||
) == ["one", "two"]
|
||||
assert wrapper.calls[0][1]["injected_job_kwargs"]["file_store"] == "archive"
|
||||
|
|
|
|||
|
|
@ -1998,7 +1998,7 @@ def test_auto_memory_reports_modified_for_create_and_false_for_skip():
|
|||
with tempfile.TemporaryDirectory() as tmpdir, temp_chdir(tmpdir):
|
||||
cwd = Path.cwd()
|
||||
app_ctx = _make_app_context(cwd)
|
||||
fs = LocalFileStore(name="test_store", embedding_store="", tag_index="default")
|
||||
fs = LocalFileStore(name="test_store", embedding_store="", tag_index="")
|
||||
wrapper = _FakeAgentWrapper()
|
||||
await fs.start()
|
||||
_install_file_jobs(app_ctx, fs)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ import pytest
|
|||
|
||||
from reme.steps.evolve._evolve import agent_reply_result_text, format_history
|
||||
from reme.steps.evolve.auto_memory import AutoMemoryStep, _sanitize_msg_for_save
|
||||
from reme.steps.evolve.auto_tag import normalize_memory_tags
|
||||
|
||||
|
||||
def test_agent_reply_result_text_uses_last_text_block():
|
||||
|
|
@ -76,26 +75,6 @@ def test_sanitize_msg_for_save_drops_tool_results_and_base64_data():
|
|||
assert sanitized.content[1].name == "memory_search"
|
||||
|
||||
|
||||
def test_auto_tag_normalizes_frontmatter_tags():
|
||||
"""Memory tags preserve entity names, de-duplicate, and stop at three."""
|
||||
assert normalize_memory_tags(
|
||||
[
|
||||
"OpenAI",
|
||||
"openai",
|
||||
"Sam Altman",
|
||||
"++",
|
||||
100,
|
||||
"宁德时代",
|
||||
"黄金",
|
||||
],
|
||||
) == ["OpenAI", "Sam_Altman", "宁德时代"]
|
||||
# pylint: disable=use-implicit-booleaness-not-comparison
|
||||
assert normalize_memory_tags(None) == []
|
||||
assert normalize_memory_tags("OpenAI") == []
|
||||
# pylint: enable=use-implicit-booleaness-not-comparison
|
||||
assert normalize_memory_tags(["x" * 65, True, {}, "宁德时代"]) == ["宁德时代"]
|
||||
|
||||
|
||||
def test_auto_memory_accepts_message_timestamp_aliases():
|
||||
"""AutoMemoryStep preserves historical message timestamps from common benchmark fields."""
|
||||
top_level = AutoMemoryStep._to_msg(
|
||||
|
|
|
|||
|
|
@ -261,11 +261,11 @@ def test_configs_define_original_jobs_without_daily_variants():
|
|||
default = resolve_app_config(config="default", log_config=False)
|
||||
assert default["jobs"]["auto_memory"]["steps"] == [
|
||||
{"backend": "auto_memory_step"},
|
||||
{"backend": "auto_tag_step", "max_tags_per_file": 3},
|
||||
{"backend": "auto_tag_step"},
|
||||
]
|
||||
assert default["jobs"]["auto_memory_cc"]["steps"] == [
|
||||
{"backend": "auto_memory_cc_step"},
|
||||
{"backend": "auto_tag_step", "max_tags_per_file": 3},
|
||||
{"backend": "auto_tag_step"},
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -56,8 +56,8 @@ def test_studio_packages_have_independent_identity() -> None:
|
|||
assert main_config["project"]["optional-dependencies"]["core"].count("reme-ai[as]") == 1
|
||||
assert main_config["project"]["optional-dependencies"]["core"].count("reme_studio") == 1
|
||||
assert "qwenpaw" not in main_config["project"]["optional-dependencies"]
|
||||
assert auto_fin_config["project"]["version"] == "0.1.2"
|
||||
assert daily_paper_config["project"]["version"] == "0.1.2"
|
||||
assert auto_fin_config["project"]["version"] == "0.1.3"
|
||||
assert daily_paper_config["project"]["version"] == "0.1.3"
|
||||
assert main_config["tool"]["setuptools"]["packages"]["find"]["include"] == ["reme", "reme.*"]
|
||||
assert "reme_studio*" in main_config["tool"]["setuptools"]["packages"]["find"]["exclude"]
|
||||
|
||||
|
|
@ -167,8 +167,8 @@ def test_auto_fin_requires_reme_base() -> None:
|
|||
|
||||
assert len(reme_requirements) == 1
|
||||
assert not reme_requirements[0].extras
|
||||
assert Version("0.4.1.8") not in reme_requirements[0].specifier
|
||||
assert Version("0.4.1.9") in reme_requirements[0].specifier
|
||||
assert Version("0.4.1.11") not in reme_requirements[0].specifier
|
||||
assert Version("0.4.1.12") in reme_requirements[0].specifier
|
||||
|
||||
|
||||
def test_daily_paper_license_matches_repository() -> None:
|
||||
|
|
@ -185,8 +185,8 @@ def test_daily_paper_declares_runtime_dependencies() -> None:
|
|||
by_name = {requirement.name: requirement for requirement in requirements}
|
||||
|
||||
assert not by_name["reme-ai"].extras
|
||||
assert Version("0.4.1.8") not in by_name["reme-ai"].specifier
|
||||
assert Version("0.4.1.9") in by_name["reme-ai"].specifier
|
||||
assert Version("0.4.1.11") not in by_name["reme-ai"].specifier
|
||||
assert Version("0.4.1.12") in by_name["reme-ai"].specifier
|
||||
assert "pypdf" in by_name
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,10 +7,12 @@ from pathlib import Path
|
|||
|
||||
import pytest
|
||||
|
||||
from reme.components.application_context import ApplicationContext
|
||||
from reme.components.file_chunker import MarkdownFileChunker
|
||||
from reme.components.file_store import LocalFileStore
|
||||
from reme.components.tag_index import LocalTagIndex
|
||||
from reme.config import resolve_app_config
|
||||
from reme.enumeration import ComponentEnum
|
||||
from reme.schema import FileChunk, FileFrontMatter, FileNode
|
||||
from reme.steps.index.list_tags import ListTagsStep
|
||||
|
||||
|
|
@ -24,6 +26,10 @@ def _chunk(chunk_id: str, path: str, text: str) -> FileChunk:
|
|||
return FileChunk(id=chunk_id, path=path, text=text, start_line=1, end_line=1)
|
||||
|
||||
|
||||
def _standalone_file_store_with_tag_index(**kwargs) -> LocalFileStore:
|
||||
return LocalFileStore(tag_index="default", **kwargs)
|
||||
|
||||
|
||||
def test_tag_normalization_and_bidirectional_mutations() -> None:
|
||||
"""Normalize FileNode tags and keep both lookup directions consistent."""
|
||||
|
||||
|
|
@ -209,6 +215,28 @@ def test_list_tags_paginates_and_applies_default_sort_orders() -> None:
|
|||
assert "empty page" in job["parameters"]["properties"]["page"]["description"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_tags_resolves_named_file_store_from_runtime_context(tmp_path) -> None:
|
||||
"""Injected tool context can route list_tags to the AutoTag Step's file store."""
|
||||
context = ApplicationContext(workspace_dir=str(tmp_path))
|
||||
default_store = LocalFileStore(name="default", embedding_store="", tag_index="")
|
||||
archive_store = LocalFileStore(name="archive", embedding_store="", tag_index="")
|
||||
default_store.tag_index = LocalTagIndex()
|
||||
archive_store.tag_index = LocalTagIndex()
|
||||
await default_store.tag_index.rebuild([_node("daily/default.md", ["default-tag"])])
|
||||
await archive_store.tag_index.rebuild([_node("daily/archive.md", ["archive-tag"])])
|
||||
context.components = {
|
||||
ComponentEnum.FILE_STORE: {
|
||||
"default": default_store,
|
||||
"archive": archive_store,
|
||||
},
|
||||
}
|
||||
|
||||
response = await ListTagsStep(app_context=context)(file_store="archive")
|
||||
|
||||
assert response.answer["items"] == [("archive-tag", 1)]
|
||||
|
||||
|
||||
def test_configured_frontmatter_key_contract() -> None:
|
||||
"""Validate, apply, and invalidate changes to the configured source key."""
|
||||
|
||||
|
|
@ -259,6 +287,55 @@ def test_configured_frontmatter_key_contract() -> None:
|
|||
assert not index.is_healthy
|
||||
|
||||
|
||||
def test_file_store_tag_index_binding_only_defaults_for_default_name() -> None:
|
||||
"""Support standalone defaults without silently substituting named dependencies."""
|
||||
default_store = LocalFileStore(name="default-store", embedding_store="", tag_index="default")
|
||||
custom_store = LocalFileStore(name="custom-store", embedding_store="", tag_index="custom")
|
||||
|
||||
default_dependency = default_store.dependency_bindings["tag_index"]
|
||||
assert default_dependency.name == "default"
|
||||
assert default_dependency.default_factory is LocalTagIndex
|
||||
assert default_dependency.optional is False
|
||||
|
||||
custom_dependency = custom_store.dependency_bindings["tag_index"]
|
||||
assert custom_dependency.name == "custom"
|
||||
assert custom_dependency.default_factory is None
|
||||
assert custom_dependency.optional is False
|
||||
|
||||
|
||||
def test_file_store_without_tag_index_name_disables_tag_index() -> None:
|
||||
"""Treat both an omitted tag-index setting and an explicit empty name as disabled."""
|
||||
omitted = LocalFileStore(name="omitted", embedding_store="")
|
||||
explicit = LocalFileStore(name="explicit", embedding_store="", tag_index="")
|
||||
|
||||
assert omitted.tag_index is None
|
||||
assert omitted.tag_index_enabled is False
|
||||
assert explicit.tag_index is None
|
||||
assert explicit.tag_index_enabled is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tag_index_name", ["", "custom", "missing"])
|
||||
def test_file_store_resolves_configured_tag_index(tag_index_name: str, tmp_path: Path) -> None:
|
||||
"""Resolve tag indexes by configured component name and fail on missing names."""
|
||||
|
||||
async def run() -> None:
|
||||
context = ApplicationContext(workspace_dir=str(tmp_path))
|
||||
index = LocalTagIndex(name="custom", tag_key="keywords")
|
||||
context.components = {ComponentEnum.TAG_INDEX: {"custom": index}}
|
||||
store = LocalFileStore(app_context=context, embedding_store="", tag_index=tag_index_name)
|
||||
|
||||
if tag_index_name == "missing":
|
||||
with pytest.raises(ValueError, match="tag_index 'missing' not found"):
|
||||
await store.start()
|
||||
assert not store.is_started
|
||||
else:
|
||||
await store._resolve_bindings()
|
||||
assert store.tag_index is (index if tag_index_name else None)
|
||||
assert store.tag_index_enabled is bool(tag_index_name)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_file_store_updates_tag_index_from_file_nodes(monkeypatch, tmp_path: Path) -> None:
|
||||
"""Keep daily and digest tags aligned through file-store mutations."""
|
||||
|
||||
|
|
@ -269,7 +346,7 @@ def test_file_store_updates_tag_index_from_file_nodes(monkeypatch, tmp_path: Pat
|
|||
|
||||
async def run() -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
store = LocalFileStore(name="test", embedding_store="", tag_index="default")
|
||||
store = _standalone_file_store_with_tag_index(name="test", embedding_store="")
|
||||
await store.start()
|
||||
assert isinstance(store.tag_index, LocalTagIndex)
|
||||
assert store.tag_index_enabled is True
|
||||
|
|
@ -304,7 +381,7 @@ def test_tag_failures_do_not_block_other_indexes_and_retry_rebuild(monkeypatch,
|
|||
|
||||
async def run() -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
store = LocalFileStore(name="test", embedding_store="", tag_index="default")
|
||||
store = _standalone_file_store_with_tag_index(name="test", embedding_store="")
|
||||
await store.start()
|
||||
assert store.tag_index_enabled
|
||||
original_rebuild = store.tag_index.rebuild
|
||||
|
|
@ -344,7 +421,7 @@ def test_failed_tag_reconciliation_makes_queries_fail_closed(monkeypatch, tmp_pa
|
|||
|
||||
async def run() -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
store = LocalFileStore(name="test", embedding_store="", tag_index="default")
|
||||
store = _standalone_file_store_with_tag_index(name="test", embedding_store="")
|
||||
await store.start()
|
||||
assert store.tag_index_enabled
|
||||
await store.upsert([(_node("daily/a.md", ["old"]), [])])
|
||||
|
|
@ -371,7 +448,7 @@ def test_tag_rebuild_graph_read_failure_does_not_block_upsert(monkeypatch, tmp_p
|
|||
|
||||
async def run() -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
store = LocalFileStore(name="test", embedding_store="", tag_index="default")
|
||||
store = _standalone_file_store_with_tag_index(name="test", embedding_store="")
|
||||
await store.start()
|
||||
assert store.tag_index_enabled
|
||||
assert store.file_graph is not None
|
||||
|
|
@ -401,7 +478,7 @@ def test_explicit_reindex_restores_tag_index(monkeypatch, tmp_path: Path) -> Non
|
|||
|
||||
async def run() -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
store = LocalFileStore(name="test", embedding_store="", tag_index="default")
|
||||
store = _standalone_file_store_with_tag_index(name="test", embedding_store="")
|
||||
await store.start()
|
||||
assert store.tag_index_enabled
|
||||
await store.upsert(
|
||||
|
|
@ -437,7 +514,7 @@ def test_tag_delete_failures_do_not_block_core_deletion(monkeypatch, tmp_path: P
|
|||
|
||||
async def run() -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
store = LocalFileStore(name="test", embedding_store="", tag_index="default")
|
||||
store = _standalone_file_store_with_tag_index(name="test", embedding_store="")
|
||||
await store.start()
|
||||
assert store.tag_index_enabled
|
||||
chunk = _chunk("chunk-a", "daily/a.md", "alpha memory")
|
||||
|
|
@ -472,7 +549,7 @@ def test_existing_markdown_chunker_supplies_frontmatter_tags(monkeypatch, tmp_pa
|
|||
note.write_text("---\nmemory_tags: [Python, ReMe]\n---\nbody\n", encoding="utf-8")
|
||||
node, chunks = await MarkdownFileChunker().chunk(note)
|
||||
|
||||
store = LocalFileStore(name="test", embedding_store="", tag_index="default")
|
||||
store = _standalone_file_store_with_tag_index(name="test", embedding_store="")
|
||||
await store.start()
|
||||
await store.upsert([(node, chunks)])
|
||||
|
||||
|
|
@ -487,14 +564,14 @@ def test_file_store_rebuilds_non_persistent_tag_index_from_graph(monkeypatch, tm
|
|||
|
||||
async def run() -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
first = LocalFileStore(name="test", embedding_store="", tag_index="default")
|
||||
first = _standalone_file_store_with_tag_index(name="test", embedding_store="")
|
||||
await first.start()
|
||||
await first.upsert([(_node("daily/a.md", ["ReMe"]), [])])
|
||||
await first.close()
|
||||
|
||||
assert not list((tmp_path / "metadata").glob("tag_index/**/*"))
|
||||
|
||||
restored = LocalFileStore(name="test", embedding_store="", tag_index="default")
|
||||
restored = _standalone_file_store_with_tag_index(name="test", embedding_store="")
|
||||
await restored.start()
|
||||
assert await restored.tag_index.paths_for_tags(["reme"]) == ["daily/a.md"]
|
||||
await restored.close()
|
||||
|
|
@ -515,9 +592,9 @@ def test_default_config_enables_tag_index_with_explicit_key() -> None:
|
|||
assert config["jobs"]["search"]["parameters"]["properties"]["tags"]["default"] == []
|
||||
assert config["jobs"]["auto_memory"]["steps"] == [
|
||||
{"backend": "auto_memory_step"},
|
||||
{"backend": "auto_tag_step", "max_tags_per_file": 3},
|
||||
{"backend": "auto_tag_step"},
|
||||
]
|
||||
assert config["jobs"]["auto_memory_cc"]["steps"] == [
|
||||
{"backend": "auto_memory_cc_step"},
|
||||
{"backend": "auto_tag_step", "max_tags_per_file": 3},
|
||||
{"backend": "auto_tag_step"},
|
||||
]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue