diff --git a/plugins/auto-fin/src/reme_auto_fin/base.py b/plugins/auto-fin/src/reme_auto_fin/base.py index 4e2de978..6eac42fe 100644 --- a/plugins/auto-fin/src/reme_auto_fin/base.py +++ b/plugins/auto-fin/src/reme_auto_fin/base.py @@ -9,11 +9,12 @@ import os from pathlib import Path import re from time import perf_counter -from typing import Any +from typing import Any, NamedTuple from uuid import uuid4 import aiofiles import frontmatter +import yaml from pydantic import BaseModel from reme.steps import BaseStep @@ -29,12 +30,30 @@ TITLE_BYTE_LIMIT = 180 _WIKILINK = re.compile(r"\[\[([^\[\]\n]+)\]\]") _HYBRID_WIKILINK = re.compile(r"(?P\[\[(?P[^\[\]\n]+)\]\])\((?P[^()\n]+)\)") _HEADING = re.compile(r"^#+\s*") -_UNSAFE_FILENAME = re.compile(r'[<>:"/\\|?*\x00-\x1f]') +# A note's stem has to survive two consumers: the filesystem, which rejects +# `<>:"/\|?*` and control characters, and `WikilinkHandler`, whose targets stop +# at `[`, `]` and `#`. Leaving the latter in place turns `- [[]]` into a +# link that resolves to some other note (a trailing `#` reads as an anchor) +# or to no link at all. +_UNSAFE_FILENAME = re.compile(r'[<>:"/\\|?*\[\]#\x00-\x1f]') FIRST_RUN_NOTICE = "今日暂无更早时段的推荐,本次为当日首次生成。" DISCLAIMER = "> 未接入可靠行情数据;本文只提供新闻研究和回顾线索,不提供收益、目标价或买卖建议。" +class WrittenReport(NamedTuple): + """One persisted note, described by what actually reached the file.""" + + path: str + """Workspace-relative POSIX path of the note.""" + + sources: list[str] + """Workspace-relative paths of the links that survived validation.""" + + body: str + """The validated body as written, with dangling wikilinks downgraded to plain text.""" + + class _TextExtractor(HTMLParser): """Flatten HTML into whitespace-separated text.""" @@ -74,7 +93,7 @@ def utc_now_iso() -> str: def normalize_title(raw: str, fallback: str) -> str: - """Return a topic or Agent label as a safe, filesystem-sized Markdown filename stem.""" + """Return a topic or Agent label as a safe, filesystem- and wikilink-safe filename stem.""" title = _UNSAFE_FILENAME.sub("-", _HEADING.sub("", str(raw or "").strip())) title = re.sub(r"\s+", " ", title).strip(" .-") if title.lower().endswith(".md"): @@ -173,11 +192,16 @@ def resolve_note_path(day_dir: Path, title: str, *, existing: Path | None) -> tu def find_note(day_dir: Path, *, kind: str, **matches: Any) -> Path | None: - """Return this workflow's note in one day directory whose frontmatter matches.""" + """Return this workflow's note in one day directory whose frontmatter matches. + + Every Markdown file in the directory is a candidate, including notes the user + is editing by hand, so an unreadable or malformed one is skipped rather than + allowed to abort the run. + """ for path in sorted(day_dir.glob("*.md")) if day_dir.is_dir() else (): try: metadata = frontmatter.load(path).metadata - except (OSError, UnicodeError, ValueError): + except (OSError, UnicodeError, ValueError, yaml.YAMLError): continue if metadata.get("kind") == kind and all(metadata.get(key) == value for key, value in matches.items()): return path @@ -190,7 +214,7 @@ def read_note(path: Path | None) -> str: return FIRST_RUN_NOTICE try: return frontmatter.load(path).content.strip()[:NOTE_CHAR_LIMIT] or FIRST_RUN_NOTICE - except (OSError, UnicodeError, ValueError): + except (OSError, UnicodeError, ValueError, yaml.YAMLError): return FIRST_RUN_NOTICE @@ -250,8 +274,8 @@ class AutoFinStep(BaseStep): existing: Path | None = None, trailer: str = "", **metadata: Any, - ) -> tuple[str, list[str]]: - """Write one report note, returning its workspace-relative path and valid sources.""" + ) -> WrittenReport: + """Write one report note, returning its path, valid sources, and validated body.""" title, path = resolve_note_path(self.day_dir, filename, existing=existing) body, sources = validate_wikilinks(normalize_hybrid_wikilinks(output.body), self.workspace_path, path) await write_markdown( @@ -269,7 +293,7 @@ class AutoFinStep(BaseStep): self._track(path, existing) relative = path.relative_to(self.workspace_path).as_posix() self.logger.info(f"[{self.name}] wrote kind={kind} path={relative} chars={len(body)} sources={len(sources)}") - return relative, sources + return WrittenReport(path=relative, sources=sources, body=body) async def _reply( self, diff --git a/plugins/auto-fin/src/reme_auto_fin/digest.py b/plugins/auto-fin/src/reme_auto_fin/digest.py index 9c9a5743..1be75795 100644 --- a/plugins/auto-fin/src/reme_auto_fin/digest.py +++ b/plugins/auto-fin/src/reme_auto_fin/digest.py @@ -32,7 +32,7 @@ class AutoFinDigestStep(AutoFinStep): earlier_brief=read_note(earlier), ), ) - path, sources = await self._write_report( + written = await self._write_report( normalize_title(f"主题新闻观察({run_date})", "主题新闻观察"), output, kind="auto-fin-digest", @@ -46,13 +46,15 @@ class AutoFinDigestStep(AutoFinStep): run_date, str(self.config_value("daily_dir")), ) - self.context["markdown_path"] = self.context["auto_fin_digest_path"] = path - self.context.response.answer = output.body + self.context["markdown_path"] = self.context["auto_fin_digest_path"] = written.path + # The caller gets the validated body, not the Agent's raw reply: a link the + # note dropped must not reappear in the API response. + self.context.response.answer = written.body self.context.response.metadata.update( { - "markdown_path": path, - "digest_path": path, - "source_paths": sources, + "markdown_path": written.path, + "digest_path": written.path, + "source_paths": written.sources, "note_paths": [note.path for note in notes], "selected_news_count": len(self._required("auto_fin_selected_news")), }, diff --git a/plugins/auto-fin/src/reme_auto_fin/research.py b/plugins/auto-fin/src/reme_auto_fin/research.py index a2bd6d32..b3c7beae 100644 --- a/plugins/auto-fin/src/reme_auto_fin/research.py +++ b/plugins/auto-fin/src/reme_auto_fin/research.py @@ -56,7 +56,7 @@ class AutoFinResearchStep(AutoFinStep): finally: if self.app_context is not None: self.app_context.metadata.get("__search_call_budgets", {}).pop(context_id, None) - path, _ = await self._write_report( + written = await self._write_report( normalize_title(topic, "主题观察"), output, kind="auto-fin-topic", @@ -65,7 +65,13 @@ class AutoFinResearchStep(AutoFinStep): topic=topic, source_news_ids=[row["news_id"] for row in recent], ) - return AutoFinNote(topic=topic, title=output.title, description=output.description, body=output.body, path=path) + return AutoFinNote( + topic=topic, + title=output.title, + description=output.description, + body=written.body, + path=written.path, + ) async def execute(self): """Write one note per topic that had relevant news, isolating per-topic failures.""" diff --git a/plugins/auto-fin/tests/test_auto_fin.py b/plugins/auto-fin/tests/test_auto_fin.py index 4b2eb207..d84d9f68 100644 --- a/plugins/auto-fin/tests/test_auto_fin.py +++ b/plugins/auto-fin/tests/test_auto_fin.py @@ -11,7 +11,14 @@ from zoneinfo import ZoneInfo import pytest import yaml -from reme_auto_fin.base import normalize_hybrid_wikilinks, normalize_title, plain_text, write_markdown +from reme_auto_fin.base import ( + FIRST_RUN_NOTICE, + normalize_hybrid_wikilinks, + normalize_title, + plain_text, + read_note, + write_markdown, +) from reme_auto_fin.data import AutoFinDataStep from reme_auto_fin.digest import AutoFinDigestStep from reme_auto_fin.research import AutoFinResearchStep @@ -20,6 +27,7 @@ from reme_auto_fin.topic import AutoFinTopicStep from reme.components import ApplicationContext from reme.components.agent_wrapper.base_agent_wrapper import BaseAgentWrapper from reme.components.runtime_context import RuntimeContext +from reme.utils.wikilink_handler import WikilinkHandler SHANGHAI = ZoneInfo("Asia/Shanghai") PLUGIN_MANIFEST = yaml.safe_load( @@ -439,6 +447,38 @@ async def test_research_keeps_the_other_topics_when_one_fails(tmp_path: Path): assert [path.name for path in (tmp_path / "daily" / "2026-08-10").glob("*.md")] == ["黄金.md"] +@pytest.mark.asyncio +async def test_research_ignores_notes_with_unparsable_frontmatter(tmp_path: Path): + """A note the user is editing by hand must not abort every topic in the run.""" + + day = tmp_path / "daily" / "2026-08-10" + day.mkdir(parents=True) + hand_edited = "---\ntags: [unfinished\n---\n\n正在编辑的笔记。\n" + (day / "手记.md").write_text(hand_edited, encoding="utf-8") + app_context = ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai") + context = _context( + auto_fin_news_by_topic={"黄金": [_news("1", "2026-08-10T09:00:00+08:00")], "机器人": [], "半导体": []}, + ) + + response = await AutoFinResearchStep(app_context=app_context, agent_wrapper=_ReportAgent(app_context=app_context))( + context, + ) + + assert response.success is True + assert response.metadata["failed_topics"] == [] + assert [item.path for item in context["auto_fin_notes"]] == ["daily/2026-08-10/黄金.md"] + assert (day / "手记.md").read_text(encoding="utf-8") == hand_edited + + +def test_read_note_falls_back_when_the_frontmatter_is_unparsable(tmp_path: Path): + broken = tmp_path / "手记.md" + broken.write_text("---\ntags: [unfinished\n---\n\n正文\n", encoding="utf-8") + + assert read_note(broken) == FIRST_RUN_NOTICE + assert read_note(tmp_path / "missing.md") == FIRST_RUN_NOTICE + assert read_note(None) == FIRST_RUN_NOTICE + + @pytest.mark.asyncio async def test_research_fails_the_run_when_every_topic_fails(tmp_path: Path): """A run that produced no note at all must fail instead of sending an empty brief.""" @@ -493,6 +533,49 @@ async def test_digest_merges_notes_and_links_back_to_each_of_them(tmp_path: Path assert response.metadata["note_paths"] == [note_path] +@pytest.mark.parametrize("topic", ["黄金", "AI[算力]", "C#", "新能源/储能", "#热点", "a|b"]) +@pytest.mark.asyncio +async def test_digest_trailer_resolves_to_each_topic_note(tmp_path: Path, topic: str): + """A topic name must not smuggle a wikilink delimiter into the trailer it lands in.""" + + app_context = ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai") + agent = _ReportAgent(app_context=app_context) + context = _context( + auto_fin_topics=[topic], + auto_fin_news_by_topic={topic: [_news("1", "2026-08-10T09:00:00+08:00")]}, + ) + await AutoFinResearchStep(app_context=app_context, agent_wrapper=agent)(context) + note_path = context["auto_fin_notes"][0].path + + response = await AutoFinDigestStep(app_context=app_context, agent_wrapper=agent)(context) + + digest = (tmp_path / response.metadata["digest_path"]).read_text(encoding="utf-8") + targets = [match.target for match in WikilinkHandler.iter_matches(digest)] + assert (tmp_path / note_path).is_file() + assert note_path in targets + # Every ``[[`` the digest emits opens a link the workspace parser can read. + assert digest.count("[[") == len(targets) + + +@pytest.mark.asyncio +async def test_digest_returns_the_validated_body_to_the_caller(tmp_path: Path): + """The API answer must not carry a link that the note itself downgraded to plain text.""" + + app_context = ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai") + agent = _ReportAgent(app_context=app_context) + context = _context( + auto_fin_news_by_topic={"黄金": [_news("1", "2026-08-10T09:00:00+08:00")], "机器人": [], "半导体": []}, + ) + await AutoFinResearchStep(app_context=app_context, agent_wrapper=agent)(context) + + response = await AutoFinDigestStep(app_context=app_context, agent_wrapper=agent)(context) + + digest = (tmp_path / response.metadata["digest_path"]).read_text(encoding="utf-8") + assert "缺失文章" in response.answer + assert "missing.md" not in response.answer + assert response.answer in digest + + @pytest.mark.asyncio async def test_digest_skips_when_the_run_was_already_skipped(tmp_path: Path): app_context = ApplicationContext(workspace_dir=str(tmp_path)) @@ -528,6 +611,16 @@ def test_normalize_title_sanitizes_agent_titles_and_keeps_notes_importable(): assert AutoFinNote(topic="黄金", title="标题", description="说明", body="正文", path="a.md").topic == "黄金" +def test_normalize_title_keeps_wikilink_delimiters_out_of_the_stem(): + """`AI[算力]` used to produce a stem the wikilink parser could not read at all.""" + + assert normalize_title("AI[算力]", "主题观察") == "AI-算力" + assert normalize_title("C#", "主题观察") == "C" + assert normalize_title("# 热点", "主题观察") == "热点" + for raw in ("AI[算力]", "C#", "# 热点", "a|b"): + assert not set("[]#|") & set(normalize_title(raw, "主题观察")) + + def test_normalize_title_fits_the_filename_component_byte_budget(): """A whole-paragraph title used to reach os.stat and fail with ENAMETOOLONG.""" assert normalize_title("长" * 200, "黄金观察") == "长" * 60