fix(auto-fin): keep hand-edited notes and wikilink delimiters from breaking the run

Addresses three review findings on the topic-per-note rework.

- `find_note` and `read_note` now skip a note whose YAML frontmatter does
  not parse. A hand-edited note in the day directory raised
  `yaml.parser.ParserError`, which per-topic isolation surfaced as
  "Auto Fin research failed for every topic" and took the run down with it.
- `normalize_title` also strips `[`, `]` and `#`, which `WikilinkHandler`
  treats as target delimiters. `AI[算力]` used to emit a trailer link the
  parser could not read at all, and `C#` resolved to `.../C` plus an anchor.
- `_write_report` returns the body it actually wrote, and both callers
  propagate it, so the digest answer and the note handed to the digest Agent
  no longer carry links that validation had already downgraded on disk.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
jinli.yl 2026-09-20 14:13:44 +08:00
parent a60289deb2
commit 7669d9dc0d
4 changed files with 143 additions and 18 deletions

View file

@ -9,11 +9,12 @@ import os
from pathlib import Path
import re
from time import perf_counter
from typing import Any
from typing import Any, NamedTuple
from uuid import uuid4
import aiofiles
import frontmatter
import yaml
from pydantic import BaseModel
from reme.steps import BaseStep
@ -29,12 +30,30 @@ TITLE_BYTE_LIMIT = 180
_WIKILINK = re.compile(r"\[\[([^\[\]\n]+)\]\]")
_HYBRID_WIKILINK = re.compile(r"(?P<wikilink>\[\[(?P<inner>[^\[\]\n]+)\]\])\((?P<destination>[^()\n]+)\)")
_HEADING = re.compile(r"^#+\s*")
_UNSAFE_FILENAME = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
# A note's stem has to survive two consumers: the filesystem, which rejects
# `<>:"/\|?*` and control characters, and `WikilinkHandler`, whose targets stop
# at `[`, `]` and `#`. Leaving the latter in place turns `- [[<path>]]` into a
# link that resolves to some other note (a trailing `#` reads as an anchor)
# or to no link at all.
_UNSAFE_FILENAME = re.compile(r'[<>:"/\\|?*\[\]#\x00-\x1f]')
FIRST_RUN_NOTICE = "今日暂无更早时段的推荐,本次为当日首次生成。"
DISCLAIMER = "> 未接入可靠行情数据;本文只提供新闻研究和回顾线索,不提供收益、目标价或买卖建议。"
class WrittenReport(NamedTuple):
"""One persisted note, described by what actually reached the file."""
path: str
"""Workspace-relative POSIX path of the note."""
sources: list[str]
"""Workspace-relative paths of the links that survived validation."""
body: str
"""The validated body as written, with dangling wikilinks downgraded to plain text."""
class _TextExtractor(HTMLParser):
"""Flatten HTML into whitespace-separated text."""
@ -74,7 +93,7 @@ def utc_now_iso() -> str:
def normalize_title(raw: str, fallback: str) -> str:
"""Return a topic or Agent label as a safe, filesystem-sized Markdown filename stem."""
"""Return a topic or Agent label as a safe, filesystem- and wikilink-safe filename stem."""
title = _UNSAFE_FILENAME.sub("-", _HEADING.sub("", str(raw or "").strip()))
title = re.sub(r"\s+", " ", title).strip(" .-")
if title.lower().endswith(".md"):
@ -173,11 +192,16 @@ def resolve_note_path(day_dir: Path, title: str, *, existing: Path | None) -> tu
def find_note(day_dir: Path, *, kind: str, **matches: Any) -> Path | None:
"""Return this workflow's note in one day directory whose frontmatter matches."""
"""Return this workflow's note in one day directory whose frontmatter matches.
Every Markdown file in the directory is a candidate, including notes the user
is editing by hand, so an unreadable or malformed one is skipped rather than
allowed to abort the run.
"""
for path in sorted(day_dir.glob("*.md")) if day_dir.is_dir() else ():
try:
metadata = frontmatter.load(path).metadata
except (OSError, UnicodeError, ValueError):
except (OSError, UnicodeError, ValueError, yaml.YAMLError):
continue
if metadata.get("kind") == kind and all(metadata.get(key) == value for key, value in matches.items()):
return path
@ -190,7 +214,7 @@ def read_note(path: Path | None) -> str:
return FIRST_RUN_NOTICE
try:
return frontmatter.load(path).content.strip()[:NOTE_CHAR_LIMIT] or FIRST_RUN_NOTICE
except (OSError, UnicodeError, ValueError):
except (OSError, UnicodeError, ValueError, yaml.YAMLError):
return FIRST_RUN_NOTICE
@ -250,8 +274,8 @@ class AutoFinStep(BaseStep):
existing: Path | None = None,
trailer: str = "",
**metadata: Any,
) -> tuple[str, list[str]]:
"""Write one report note, returning its workspace-relative path and valid sources."""
) -> WrittenReport:
"""Write one report note, returning its path, valid sources, and validated body."""
title, path = resolve_note_path(self.day_dir, filename, existing=existing)
body, sources = validate_wikilinks(normalize_hybrid_wikilinks(output.body), self.workspace_path, path)
await write_markdown(
@ -269,7 +293,7 @@ class AutoFinStep(BaseStep):
self._track(path, existing)
relative = path.relative_to(self.workspace_path).as_posix()
self.logger.info(f"[{self.name}] wrote kind={kind} path={relative} chars={len(body)} sources={len(sources)}")
return relative, sources
return WrittenReport(path=relative, sources=sources, body=body)
async def _reply(
self,

View file

@ -32,7 +32,7 @@ class AutoFinDigestStep(AutoFinStep):
earlier_brief=read_note(earlier),
),
)
path, sources = await self._write_report(
written = await self._write_report(
normalize_title(f"主题新闻观察({run_date})", "主题新闻观察"),
output,
kind="auto-fin-digest",
@ -46,13 +46,15 @@ class AutoFinDigestStep(AutoFinStep):
run_date,
str(self.config_value("daily_dir")),
)
self.context["markdown_path"] = self.context["auto_fin_digest_path"] = path
self.context.response.answer = output.body
self.context["markdown_path"] = self.context["auto_fin_digest_path"] = written.path
# The caller gets the validated body, not the Agent's raw reply: a link the
# note dropped must not reappear in the API response.
self.context.response.answer = written.body
self.context.response.metadata.update(
{
"markdown_path": path,
"digest_path": path,
"source_paths": sources,
"markdown_path": written.path,
"digest_path": written.path,
"source_paths": written.sources,
"note_paths": [note.path for note in notes],
"selected_news_count": len(self._required("auto_fin_selected_news")),
},

View file

@ -56,7 +56,7 @@ class AutoFinResearchStep(AutoFinStep):
finally:
if self.app_context is not None:
self.app_context.metadata.get("__search_call_budgets", {}).pop(context_id, None)
path, _ = await self._write_report(
written = await self._write_report(
normalize_title(topic, "主题观察"),
output,
kind="auto-fin-topic",
@ -65,7 +65,13 @@ class AutoFinResearchStep(AutoFinStep):
topic=topic,
source_news_ids=[row["news_id"] for row in recent],
)
return AutoFinNote(topic=topic, title=output.title, description=output.description, body=output.body, path=path)
return AutoFinNote(
topic=topic,
title=output.title,
description=output.description,
body=written.body,
path=written.path,
)
async def execute(self):
"""Write one note per topic that had relevant news, isolating per-topic failures."""

View file

@ -11,7 +11,14 @@ from zoneinfo import ZoneInfo
import pytest
import yaml
from reme_auto_fin.base import normalize_hybrid_wikilinks, normalize_title, plain_text, write_markdown
from reme_auto_fin.base import (
FIRST_RUN_NOTICE,
normalize_hybrid_wikilinks,
normalize_title,
plain_text,
read_note,
write_markdown,
)
from reme_auto_fin.data import AutoFinDataStep
from reme_auto_fin.digest import AutoFinDigestStep
from reme_auto_fin.research import AutoFinResearchStep
@ -20,6 +27,7 @@ from reme_auto_fin.topic import AutoFinTopicStep
from reme.components import ApplicationContext
from reme.components.agent_wrapper.base_agent_wrapper import BaseAgentWrapper
from reme.components.runtime_context import RuntimeContext
from reme.utils.wikilink_handler import WikilinkHandler
SHANGHAI = ZoneInfo("Asia/Shanghai")
PLUGIN_MANIFEST = yaml.safe_load(
@ -439,6 +447,38 @@ async def test_research_keeps_the_other_topics_when_one_fails(tmp_path: Path):
assert [path.name for path in (tmp_path / "daily" / "2026-08-10").glob("*.md")] == ["黄金.md"]
@pytest.mark.asyncio
async def test_research_ignores_notes_with_unparsable_frontmatter(tmp_path: Path):
"""A note the user is editing by hand must not abort every topic in the run."""
day = tmp_path / "daily" / "2026-08-10"
day.mkdir(parents=True)
hand_edited = "---\ntags: [unfinished\n---\n\n正在编辑的笔记。\n"
(day / "手记.md").write_text(hand_edited, encoding="utf-8")
app_context = ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai")
context = _context(
auto_fin_news_by_topic={"黄金": [_news("1", "2026-08-10T09:00:00+08:00")], "机器人": [], "半导体": []},
)
response = await AutoFinResearchStep(app_context=app_context, agent_wrapper=_ReportAgent(app_context=app_context))(
context,
)
assert response.success is True
assert response.metadata["failed_topics"] == []
assert [item.path for item in context["auto_fin_notes"]] == ["daily/2026-08-10/黄金.md"]
assert (day / "手记.md").read_text(encoding="utf-8") == hand_edited
def test_read_note_falls_back_when_the_frontmatter_is_unparsable(tmp_path: Path):
broken = tmp_path / "手记.md"
broken.write_text("---\ntags: [unfinished\n---\n\n正文\n", encoding="utf-8")
assert read_note(broken) == FIRST_RUN_NOTICE
assert read_note(tmp_path / "missing.md") == FIRST_RUN_NOTICE
assert read_note(None) == FIRST_RUN_NOTICE
@pytest.mark.asyncio
async def test_research_fails_the_run_when_every_topic_fails(tmp_path: Path):
"""A run that produced no note at all must fail instead of sending an empty brief."""
@ -493,6 +533,49 @@ async def test_digest_merges_notes_and_links_back_to_each_of_them(tmp_path: Path
assert response.metadata["note_paths"] == [note_path]
@pytest.mark.parametrize("topic", ["黄金", "AI[算力]", "C#", "新能源/储能", "#热点", "a|b"])
@pytest.mark.asyncio
async def test_digest_trailer_resolves_to_each_topic_note(tmp_path: Path, topic: str):
"""A topic name must not smuggle a wikilink delimiter into the trailer it lands in."""
app_context = ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai")
agent = _ReportAgent(app_context=app_context)
context = _context(
auto_fin_topics=[topic],
auto_fin_news_by_topic={topic: [_news("1", "2026-08-10T09:00:00+08:00")]},
)
await AutoFinResearchStep(app_context=app_context, agent_wrapper=agent)(context)
note_path = context["auto_fin_notes"][0].path
response = await AutoFinDigestStep(app_context=app_context, agent_wrapper=agent)(context)
digest = (tmp_path / response.metadata["digest_path"]).read_text(encoding="utf-8")
targets = [match.target for match in WikilinkHandler.iter_matches(digest)]
assert (tmp_path / note_path).is_file()
assert note_path in targets
# Every ``[[`` the digest emits opens a link the workspace parser can read.
assert digest.count("[[") == len(targets)
@pytest.mark.asyncio
async def test_digest_returns_the_validated_body_to_the_caller(tmp_path: Path):
"""The API answer must not carry a link that the note itself downgraded to plain text."""
app_context = ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai")
agent = _ReportAgent(app_context=app_context)
context = _context(
auto_fin_news_by_topic={"黄金": [_news("1", "2026-08-10T09:00:00+08:00")], "机器人": [], "半导体": []},
)
await AutoFinResearchStep(app_context=app_context, agent_wrapper=agent)(context)
response = await AutoFinDigestStep(app_context=app_context, agent_wrapper=agent)(context)
digest = (tmp_path / response.metadata["digest_path"]).read_text(encoding="utf-8")
assert "缺失文章" in response.answer
assert "missing.md" not in response.answer
assert response.answer in digest
@pytest.mark.asyncio
async def test_digest_skips_when_the_run_was_already_skipped(tmp_path: Path):
app_context = ApplicationContext(workspace_dir=str(tmp_path))
@ -528,6 +611,16 @@ def test_normalize_title_sanitizes_agent_titles_and_keeps_notes_importable():
assert AutoFinNote(topic="黄金", title="标题", description="说明", body="正文", path="a.md").topic == "黄金"
def test_normalize_title_keeps_wikilink_delimiters_out_of_the_stem():
"""`AI[算力]` used to produce a stem the wikilink parser could not read at all."""
assert normalize_title("AI[算力]", "主题观察") == "AI-算力"
assert normalize_title("C#", "主题观察") == "C"
assert normalize_title("# 热点", "主题观察") == "热点"
for raw in ("AI[算力]", "C#", "# 热点", "a|b"):
assert not set("[]#|") & set(normalize_title(raw, "主题观察"))
def test_normalize_title_fits_the_filename_component_byte_budget():
"""A whole-paragraph title used to reach os.stat and fail with ENAMETOOLONG."""
assert normalize_title("长" * 200, "黄金观察") == "长" * 60