From 7669d9dc0d33cadddc66e08afa1378a67439c683 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Sun, 20 Sep 2026 14:13:44 +0800 Subject: [PATCH] fix(auto-fin): keep hand-edited notes and wikilink delimiters from breaking the run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses three review findings on the topic-per-note rework. - `find_note` and `read_note` now skip a note whose YAML frontmatter does not parse. A hand-edited note in the day directory raised `yaml.parser.ParserError`, which per-topic isolation surfaced as "Auto Fin research failed for every topic" and took the run down with it. - `normalize_title` also strips `[`, `]` and `#`, which `WikilinkHandler` treats as target delimiters. `AI[算力]` used to emit a trailer link the parser could not read at all, and `C#` resolved to `.../C` plus an anchor. - `_write_report` returns the body it actually wrote, and both callers propagate it, so the digest answer and the note handed to the digest Agent no longer carry links that validation had already downgraded on disk. Co-Authored-By: Claude --- plugins/auto-fin/src/reme_auto_fin/base.py | 42 ++++++-- plugins/auto-fin/src/reme_auto_fin/digest.py | 14 +-- .../auto-fin/src/reme_auto_fin/research.py | 10 +- plugins/auto-fin/tests/test_auto_fin.py | 95 ++++++++++++++++++- 4 files changed, 143 insertions(+), 18 deletions(-) diff --git a/plugins/auto-fin/src/reme_auto_fin/base.py b/plugins/auto-fin/src/reme_auto_fin/base.py index 4e2de978..6eac42fe 100644 --- a/plugins/auto-fin/src/reme_auto_fin/base.py +++ b/plugins/auto-fin/src/reme_auto_fin/base.py @@ -9,11 +9,12 @@ import os from pathlib import Path import re from time import perf_counter -from typing import Any +from typing import Any, NamedTuple from uuid import uuid4 import aiofiles import frontmatter +import yaml from pydantic import BaseModel from reme.steps import BaseStep @@ -29,12 +30,30 @@ TITLE_BYTE_LIMIT = 180 _WIKILINK = re.compile(r"\[\[([^\[\]\n]+)\]\]") _HYBRID_WIKILINK = re.compile(r"(?P\[\[(?P[^\[\]\n]+)\]\])\((?P[^()\n]+)\)") _HEADING = re.compile(r"^#+\s*") -_UNSAFE_FILENAME = re.compile(r'[<>:"/\\|?*\x00-\x1f]') +# A note's stem has to survive two consumers: the filesystem, which rejects +# `<>:"/\|?*` and control characters, and `WikilinkHandler`, whose targets stop +# at `[`, `]` and `#`. Leaving the latter in place turns `- [[]]` into a +# link that resolves to some other note (a trailing `#` reads as an anchor) +# or to no link at all. +_UNSAFE_FILENAME = re.compile(r'[<>:"/\\|?*\[\]#\x00-\x1f]') FIRST_RUN_NOTICE = "今日暂无更早时段的推荐,本次为当日首次生成。" DISCLAIMER = "> 未接入可靠行情数据;本文只提供新闻研究和回顾线索,不提供收益、目标价或买卖建议。" +class WrittenReport(NamedTuple): + """One persisted note, described by what actually reached the file.""" + + path: str + """Workspace-relative POSIX path of the note.""" + + sources: list[str] + """Workspace-relative paths of the links that survived validation.""" + + body: str + """The validated body as written, with dangling wikilinks downgraded to plain text.""" + + class _TextExtractor(HTMLParser): """Flatten HTML into whitespace-separated text.""" @@ -74,7 +93,7 @@ def utc_now_iso() -> str: def normalize_title(raw: str, fallback: str) -> str: - """Return a topic or Agent label as a safe, filesystem-sized Markdown filename stem.""" + """Return a topic or Agent label as a safe, filesystem- and wikilink-safe filename stem.""" title = _UNSAFE_FILENAME.sub("-", _HEADING.sub("", str(raw or "").strip())) title = re.sub(r"\s+", " ", title).strip(" .-") if title.lower().endswith(".md"): @@ -173,11 +192,16 @@ def resolve_note_path(day_dir: Path, title: str, *, existing: Path | None) -> tu def find_note(day_dir: Path, *, kind: str, **matches: Any) -> Path | None: - """Return this workflow's note in one day directory whose frontmatter matches.""" + """Return this workflow's note in one day directory whose frontmatter matches. + + Every Markdown file in the directory is a candidate, including notes the user + is editing by hand, so an unreadable or malformed one is skipped rather than + allowed to abort the run. + """ for path in sorted(day_dir.glob("*.md")) if day_dir.is_dir() else (): try: metadata = frontmatter.load(path).metadata - except (OSError, UnicodeError, ValueError): + except (OSError, UnicodeError, ValueError, yaml.YAMLError): continue if metadata.get("kind") == kind and all(metadata.get(key) == value for key, value in matches.items()): return path @@ -190,7 +214,7 @@ def read_note(path: Path | None) -> str: return FIRST_RUN_NOTICE try: return frontmatter.load(path).content.strip()[:NOTE_CHAR_LIMIT] or FIRST_RUN_NOTICE - except (OSError, UnicodeError, ValueError): + except (OSError, UnicodeError, ValueError, yaml.YAMLError): return FIRST_RUN_NOTICE @@ -250,8 +274,8 @@ class AutoFinStep(BaseStep): existing: Path | None = None, trailer: str = "", **metadata: Any, - ) -> tuple[str, list[str]]: - """Write one report note, returning its workspace-relative path and valid sources.""" + ) -> WrittenReport: + """Write one report note, returning its path, valid sources, and validated body.""" title, path = resolve_note_path(self.day_dir, filename, existing=existing) body, sources = validate_wikilinks(normalize_hybrid_wikilinks(output.body), self.workspace_path, path) await write_markdown( @@ -269,7 +293,7 @@ class AutoFinStep(BaseStep): self._track(path, existing) relative = path.relative_to(self.workspace_path).as_posix() self.logger.info(f"[{self.name}] wrote kind={kind} path={relative} chars={len(body)} sources={len(sources)}") - return relative, sources + return WrittenReport(path=relative, sources=sources, body=body) async def _reply( self, diff --git a/plugins/auto-fin/src/reme_auto_fin/digest.py b/plugins/auto-fin/src/reme_auto_fin/digest.py index 9c9a5743..1be75795 100644 --- a/plugins/auto-fin/src/reme_auto_fin/digest.py +++ b/plugins/auto-fin/src/reme_auto_fin/digest.py @@ -32,7 +32,7 @@ class AutoFinDigestStep(AutoFinStep): earlier_brief=read_note(earlier), ), ) - path, sources = await self._write_report( + written = await self._write_report( normalize_title(f"主题新闻观察({run_date})", "主题新闻观察"), output, kind="auto-fin-digest", @@ -46,13 +46,15 @@ class AutoFinDigestStep(AutoFinStep): run_date, str(self.config_value("daily_dir")), ) - self.context["markdown_path"] = self.context["auto_fin_digest_path"] = path - self.context.response.answer = output.body + self.context["markdown_path"] = self.context["auto_fin_digest_path"] = written.path + # The caller gets the validated body, not the Agent's raw reply: a link the + # note dropped must not reappear in the API response. + self.context.response.answer = written.body self.context.response.metadata.update( { - "markdown_path": path, - "digest_path": path, - "source_paths": sources, + "markdown_path": written.path, + "digest_path": written.path, + "source_paths": written.sources, "note_paths": [note.path for note in notes], "selected_news_count": len(self._required("auto_fin_selected_news")), }, diff --git a/plugins/auto-fin/src/reme_auto_fin/research.py b/plugins/auto-fin/src/reme_auto_fin/research.py index a2bd6d32..b3c7beae 100644 --- a/plugins/auto-fin/src/reme_auto_fin/research.py +++ b/plugins/auto-fin/src/reme_auto_fin/research.py @@ -56,7 +56,7 @@ class AutoFinResearchStep(AutoFinStep): finally: if self.app_context is not None: self.app_context.metadata.get("__search_call_budgets", {}).pop(context_id, None) - path, _ = await self._write_report( + written = await self._write_report( normalize_title(topic, "主题观察"), output, kind="auto-fin-topic", @@ -65,7 +65,13 @@ class AutoFinResearchStep(AutoFinStep): topic=topic, source_news_ids=[row["news_id"] for row in recent], ) - return AutoFinNote(topic=topic, title=output.title, description=output.description, body=output.body, path=path) + return AutoFinNote( + topic=topic, + title=output.title, + description=output.description, + body=written.body, + path=written.path, + ) async def execute(self): """Write one note per topic that had relevant news, isolating per-topic failures.""" diff --git a/plugins/auto-fin/tests/test_auto_fin.py b/plugins/auto-fin/tests/test_auto_fin.py index 4b2eb207..d84d9f68 100644 --- a/plugins/auto-fin/tests/test_auto_fin.py +++ b/plugins/auto-fin/tests/test_auto_fin.py @@ -11,7 +11,14 @@ from zoneinfo import ZoneInfo import pytest import yaml -from reme_auto_fin.base import normalize_hybrid_wikilinks, normalize_title, plain_text, write_markdown +from reme_auto_fin.base import ( + FIRST_RUN_NOTICE, + normalize_hybrid_wikilinks, + normalize_title, + plain_text, + read_note, + write_markdown, +) from reme_auto_fin.data import AutoFinDataStep from reme_auto_fin.digest import AutoFinDigestStep from reme_auto_fin.research import AutoFinResearchStep @@ -20,6 +27,7 @@ from reme_auto_fin.topic import AutoFinTopicStep from reme.components import ApplicationContext from reme.components.agent_wrapper.base_agent_wrapper import BaseAgentWrapper from reme.components.runtime_context import RuntimeContext +from reme.utils.wikilink_handler import WikilinkHandler SHANGHAI = ZoneInfo("Asia/Shanghai") PLUGIN_MANIFEST = yaml.safe_load( @@ -439,6 +447,38 @@ async def test_research_keeps_the_other_topics_when_one_fails(tmp_path: Path): assert [path.name for path in (tmp_path / "daily" / "2026-08-10").glob("*.md")] == ["黄金.md"] +@pytest.mark.asyncio +async def test_research_ignores_notes_with_unparsable_frontmatter(tmp_path: Path): + """A note the user is editing by hand must not abort every topic in the run.""" + + day = tmp_path / "daily" / "2026-08-10" + day.mkdir(parents=True) + hand_edited = "---\ntags: [unfinished\n---\n\n正在编辑的笔记。\n" + (day / "手记.md").write_text(hand_edited, encoding="utf-8") + app_context = ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai") + context = _context( + auto_fin_news_by_topic={"黄金": [_news("1", "2026-08-10T09:00:00+08:00")], "机器人": [], "半导体": []}, + ) + + response = await AutoFinResearchStep(app_context=app_context, agent_wrapper=_ReportAgent(app_context=app_context))( + context, + ) + + assert response.success is True + assert response.metadata["failed_topics"] == [] + assert [item.path for item in context["auto_fin_notes"]] == ["daily/2026-08-10/黄金.md"] + assert (day / "手记.md").read_text(encoding="utf-8") == hand_edited + + +def test_read_note_falls_back_when_the_frontmatter_is_unparsable(tmp_path: Path): + broken = tmp_path / "手记.md" + broken.write_text("---\ntags: [unfinished\n---\n\n正文\n", encoding="utf-8") + + assert read_note(broken) == FIRST_RUN_NOTICE + assert read_note(tmp_path / "missing.md") == FIRST_RUN_NOTICE + assert read_note(None) == FIRST_RUN_NOTICE + + @pytest.mark.asyncio async def test_research_fails_the_run_when_every_topic_fails(tmp_path: Path): """A run that produced no note at all must fail instead of sending an empty brief.""" @@ -493,6 +533,49 @@ async def test_digest_merges_notes_and_links_back_to_each_of_them(tmp_path: Path assert response.metadata["note_paths"] == [note_path] +@pytest.mark.parametrize("topic", ["黄金", "AI[算力]", "C#", "新能源/储能", "#热点", "a|b"]) +@pytest.mark.asyncio +async def test_digest_trailer_resolves_to_each_topic_note(tmp_path: Path, topic: str): + """A topic name must not smuggle a wikilink delimiter into the trailer it lands in.""" + + app_context = ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai") + agent = _ReportAgent(app_context=app_context) + context = _context( + auto_fin_topics=[topic], + auto_fin_news_by_topic={topic: [_news("1", "2026-08-10T09:00:00+08:00")]}, + ) + await AutoFinResearchStep(app_context=app_context, agent_wrapper=agent)(context) + note_path = context["auto_fin_notes"][0].path + + response = await AutoFinDigestStep(app_context=app_context, agent_wrapper=agent)(context) + + digest = (tmp_path / response.metadata["digest_path"]).read_text(encoding="utf-8") + targets = [match.target for match in WikilinkHandler.iter_matches(digest)] + assert (tmp_path / note_path).is_file() + assert note_path in targets + # Every ``[[`` the digest emits opens a link the workspace parser can read. + assert digest.count("[[") == len(targets) + + +@pytest.mark.asyncio +async def test_digest_returns_the_validated_body_to_the_caller(tmp_path: Path): + """The API answer must not carry a link that the note itself downgraded to plain text.""" + + app_context = ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai") + agent = _ReportAgent(app_context=app_context) + context = _context( + auto_fin_news_by_topic={"黄金": [_news("1", "2026-08-10T09:00:00+08:00")], "机器人": [], "半导体": []}, + ) + await AutoFinResearchStep(app_context=app_context, agent_wrapper=agent)(context) + + response = await AutoFinDigestStep(app_context=app_context, agent_wrapper=agent)(context) + + digest = (tmp_path / response.metadata["digest_path"]).read_text(encoding="utf-8") + assert "缺失文章" in response.answer + assert "missing.md" not in response.answer + assert response.answer in digest + + @pytest.mark.asyncio async def test_digest_skips_when_the_run_was_already_skipped(tmp_path: Path): app_context = ApplicationContext(workspace_dir=str(tmp_path)) @@ -528,6 +611,16 @@ def test_normalize_title_sanitizes_agent_titles_and_keeps_notes_importable(): assert AutoFinNote(topic="黄金", title="标题", description="说明", body="正文", path="a.md").topic == "黄金" +def test_normalize_title_keeps_wikilink_delimiters_out_of_the_stem(): + """`AI[算力]` used to produce a stem the wikilink parser could not read at all.""" + + assert normalize_title("AI[算力]", "主题观察") == "AI-算力" + assert normalize_title("C#", "主题观察") == "C" + assert normalize_title("# 热点", "主题观察") == "热点" + for raw in ("AI[算力]", "C#", "# 热点", "a|b"): + assert not set("[]#|") & set(normalize_title(raw, "主题观察")) + + def test_normalize_title_fits_the_filename_component_byte_budget(): """A whole-paragraph title used to reach os.stat and fail with ENAMETOOLONG.""" assert normalize_title("长" * 200, "黄金观察") == "长" * 60