From 4cc9a1c292f8bb921da8ad73211afc160952f4ea Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Fri, 18 Sep 2026 23:44:20 +0800 Subject: [PATCH] refactor(auto-fin): write one note per topic before the daily digest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge step did two jobs at once: it researched every topic and combined the results into a single report. Split it the way daily-paper separates analysis from its brief, so each topic earns a durable note of its own. - auto_fin_research_step writes one note per topic that had relevant news, tagged `kind: auto-fin-topic` and `topic` in frontmatter - auto_fin_digest_step merges those notes into the day's brief with no tools of its own and appends a `## 主题详解` section linking back to each note - a same-day rerun finds a topic's note by its `topic` frontmatter and replaces it in place, deleting the old file when the title changed - base.py now owns the shared Markdown layer: title sanitizing, report normalizing, wikilink validation, note lookup, atomic frontmatter writes, and change tracking, so both steps share one write path - the DingTalk step maps `auto_fin_digest_path` to `markdown_path` explicitly instead of relying on whichever step ran last - drop the unused AutoFinTopicOutput schema and read `job_tools` from the step config rather than hardcoding `search` Co-Authored-By: Claude --- plugins/auto-fin/README.md | 50 +-- plugins/auto-fin/README_ZH.md | 31 +- .../auto-fin/src/reme_auto_fin/__init__.py | 12 +- plugins/auto-fin/src/reme_auto_fin/base.py | 220 ++++++++++++- plugins/auto-fin/src/reme_auto_fin/data.py | 6 +- plugins/auto-fin/src/reme_auto_fin/digest.py | 62 ++++ .../auto-fin/src/reme_auto_fin/digest.yaml | 20 ++ plugins/auto-fin/src/reme_auto_fin/merge.py | 190 ----------- .../auto-fin/src/reme_auto_fin/plugin.yaml | 6 +- .../auto-fin/src/reme_auto_fin/research.py | 93 ++++++ .../{merge.yaml => research.yaml} | 20 +- plugins/auto-fin/src/reme_auto_fin/schema.py | 12 +- plugins/auto-fin/tests/test_auto_fin.py | 303 +++++++++++------- reme/config/cookbook.yaml | 7 +- tests/unit/test_cookbook_config.py | 6 +- 15 files changed, 651 insertions(+), 387 deletions(-) create mode 100644 plugins/auto-fin/src/reme_auto_fin/digest.py create mode 100644 plugins/auto-fin/src/reme_auto_fin/digest.yaml delete mode 100644 plugins/auto-fin/src/reme_auto_fin/merge.py create mode 100644 plugins/auto-fin/src/reme_auto_fin/research.py rename plugins/auto-fin/src/reme_auto_fin/{merge.yaml => research.yaml} (61%) diff --git a/plugins/auto-fin/README.md b/plugins/auto-fin/README.md index 35edbd82..5f007478 100644 --- a/plugins/auto-fin/README.md +++ b/plugins/auto-fin/README.md @@ -3,11 +3,11 @@ [中文](README_ZH.md) Auto Fin fetches a rolling window of CLS telegraph news (24 hours by default), groups items by configured -topics, researches each topic against ReMe history, and writes one Chinese Markdown report with validated wikilinks. -Current news and topic selection stay in runtime memory; only the final report becomes durable memory. This directory -is an independent Python distribution. Its single `reme.plugins` entry point exposes a `plugin.yaml` containing the -three Step backends and their Job configuration under `application_defaults`. Enable the installed plugin explicitly -through `plugins=["auto-fin"]`. +topics, researches each topic against ReMe history into its own note, and merges those notes into one dated brief +whose validated wikilinks point back at every note. Current news and topic selection stay in runtime memory; only the +notes and the brief become durable memory. This directory is an independent Python distribution. Its single +`reme.plugins` entry point exposes a `plugin.yaml` containing the four Step backends and their Job configuration under +`application_defaults`. Enable the installed plugin explicitly through `plugins=["auto-fin"]`. > Auto Fin has no reliable market-price feed. It does not calculate returns, targets, or entry points and is not > investment advice. @@ -72,11 +72,10 @@ normalize and deduplicate in RuntimeContext ↓ topic Agent returns topic-to-news-ID mappings in prompt-sized batches ↓ -one research Agent per topic examines its latest 20 articles and searches history up to three times +one research Agent per topic examines its latest 20 articles, searches history up to three times, +and writes one note per topic with validated historical wikilinks ↓ -combine topic results and validate historical wikilinks in code - ↓ -daily/YYYY-MM-DD/auto_fin.md +digest Agent merges the notes into today's brief; code appends a link back to every note ↓ generate memory tags; the background file watcher refreshes indexes ``` @@ -89,16 +88,20 @@ records outside the window are discarded. values for each topic. Code ignores unknown IDs and deduplicates repeated IDs, then preserves source-news order. One article may belong to multiple topics. If nothing is relevant, the job succeeds as a skip without writing a report. -`auto_fin_merge_step` researches each nonempty topic with its latest 20 articles. It exposes only `search`, enforces a three-call search budget per topic, and keeps current CLS IDs, -times, and titles as plain evidence. The prompt limits -wikilinks to historical Markdown actually used by the Agent; the code-level boundary independently keeps only existing, -workspace-relative Markdown targets. Missing, absolute, escaping, backslash, and self-referential targets are degraded -to their readable aliases. +`auto_fin_research_step` researches each nonempty topic with its latest 20 articles and writes one note per topic. It +exposes only `search`, enforces a three-call search budget per topic, and keeps current CLS IDs, times, and titles as +plain evidence. The prompt limits wikilinks to historical Markdown actually used by the Agent; the code-level boundary +independently keeps only existing, workspace-relative Markdown targets. Missing, absolute, escaping, backslash, and +self-referential targets are degraded to their readable aliases. -Same-day reruns use up to the first 30,000 characters of the existing report as context and replace it with the revised result. The combined final write is atomic and -refreshes the daily index. The workflow then runs `auto_tag_step` to update the generated report's memory-tag -frontmatter; the normal background file watcher observes that source-file change and refreshes derived indexes. No -JSONL, intermediate Markdown, or structured Agent output is written. +`auto_fin_digest_step` merges those notes into today's brief with no tools of its own: the notes already carry the +research. Code appends a `## 主题详解` section linking back to every note, refreshes the daily index, and publishes the +brief path for downstream steps. Both steps stop early when the topic step found no relevant news. + +Same-day reruns reuse the note a topic already produced, feeding its body back to the research Agent and replacing it +in place; the brief is replaced the same way. Writes are atomic. The workflow then runs `auto_tag_step` to update the +generated files' memory-tag frontmatter; the normal background file watcher observes those source-file changes and +refreshes derived indexes. No JSONL, intermediate Markdown, or structured Agent output is written. ## Parameters @@ -116,12 +119,15 @@ The plugin cron Job starts with the application and runs daily at 09:00 in the a ## Output ```text -.reme/daily/YYYY-MM-DD/auto_fin.md +.reme/daily/YYYY-MM-DD/.md # one per topic that had relevant news +.reme/daily/YYYY-MM-DD/.md # today's merged brief, linking back to every note ``` -The report includes a title, description, current CLS evidence, historical analysis, contextual wikilinks, and a fixed -non-investment disclaimer. Network errors and invalid Agent output fail explicitly; no relevant current news is a -successful skip. +Every file carries `kind` frontmatter (`auto-fin-topic` or `auto-fin-digest`), so a rerun finds and replaces the notes +it produced instead of duplicating them. Each file includes a title, description, current CLS evidence, historical +analysis, contextual wikilinks, and a fixed non-investment disclaimer; the brief ends with a `## 主题详解` list linking +to the topic notes. Network errors and invalid Agent output fail explicitly; no relevant current news is a successful +skip. ## Validation diff --git a/plugins/auto-fin/README_ZH.md b/plugins/auto-fin/README_ZH.md index 21c08771..2ebf9348 100644 --- a/plugins/auto-fin/README_ZH.md +++ b/plugins/auto-fin/README_ZH.md @@ -2,9 +2,9 @@ [English](README.md) -Auto Fin 自动拉取一个滚动时间窗口内的财联社电报(默认 24 小时),按配置 topics 归类相关新闻,逐主题搜索 ReMe 中有回顾价值的历史材料,最后写入一份带校验 -wikilink 的中文 Markdown 报告。当前新闻和筛选结果只存在于本次运行内存中,只有最终报告成为持久记忆。本目录是一个独立 Python -distribution:单个 `reme.plugins` entry point 暴露 `plugin.yaml`,其中声明三个 Step backend,并在 +Auto Fin 自动拉取一个滚动时间窗口内的财联社电报(默认 24 小时),按配置 topics 归类相关新闻,逐主题搜索 ReMe 中有回顾价值的历史材料并为每个主题写一份笔记, +最后把所有主题笔记合并成一份带校验 wikilink、且回链到各主题笔记的当日总览。当前新闻和筛选结果只存在于本次运行内存中,只有主题笔记和总览成为持久记忆。本目录是一个独立 Python +distribution:单个 `reme.plugins` entry point 暴露 `plugin.yaml`,其中声明四个 Step backend,并在 `application_defaults` 下提供 Job 配置;通过 `plugins=["auto-fin"]` 显式启用这个已安装插件。 > Auto Fin 没有可靠行情数据,不计算收益、目标价或买卖点,也不提供投资建议。 @@ -67,11 +67,10 @@ Auto Fin 和自动标签使用的 `search`、`list_tags`、`frontmatter_read` ↓ Topic Agent 按提示词长度分批输出“主题 → news_id” ↓ -每个有新闻的主题由独立 Research Agent 研究最新 20 篇;每个 Agent 最多搜索 3 次 +每个有新闻的主题由独立 Research Agent 研究最新 20 篇,最多搜索 3 次, +并写出一份带校验历史 wikilink 的主题笔记 ↓ -合并各主题结果并校验历史 wikilink - ↓ -daily/YYYY-MM-DD/auto_fin.md +Digest Agent 把各主题笔记合并成当日总览,代码在末尾追加回链到每一份笔记 ↓ 生成记忆标签,由后台文件 watcher 刷新索引 ``` @@ -80,14 +79,17 @@ daily/YYYY-MM-DD/auto_fin.md 小时。请求带有限速和重试;损坏记录及窗口外记录会被丢弃。 `auto_fin_topic_step` 按完整提示词的 10 万字符上限分批接收当前新闻,返回每个主题的相关 `news_id`。代码会忽略未知 ID、去除重复 ID,并保持源新闻顺序;一条新闻可属于多个主题。如果没有相关新闻,Job -会成功跳过,不写报告。 +会成功跳过,不写任何文件。 -`auto_fin_merge_step` 依次研究每个有新闻的主题,只送入该主题最新的 20 篇新闻,并向 Agent 仅开放 `search`;代码按主题限制最多 3 次搜索。当前新闻以 CLS ID、时间和标题作为普通证据。 +`auto_fin_research_step` 为每个有新闻的主题研究最新 20 篇,并各写一份主题笔记;只向 Agent 开放 `search`,按主题限制最多 3 次搜索。当前新闻以 CLS ID、时间和标题作为普通证据。 Prompt 要求 Agent 只链接实际使用过的历史 Markdown;代码边界则独立保证只保留真实存在、相对 workspace 的 Markdown 目标。不存在、绝对路径、越界、带反斜杠和自引用的目标都会降级为可读 alias。 -同日重跑会参考当天已有报告(最多前 3 万字符)并覆盖为修订结果。各主题结果合并后原子写入并刷新当天索引,随后通过 `auto_tag_step` -更新报告的记忆标签 frontmatter;常规后台文件 watcher 会观察该源文件变化并刷新派生索引。流程不会写入 JSONL、 +`auto_fin_digest_step` 把各主题笔记合并成当日总览,自身不开放任何工具——研究结论已经在笔记里。代码在正文末尾追加 `## 主题详解` +章节,回链到每一份主题笔记,然后刷新当天索引并把总览路径交给下游步骤。两步在工作流已被判定跳过时都会提前返回。 + +同日重跑会按 frontmatter 中的 `topic` 找回该主题已有的笔记,把它的正文作为上下文重新研究并原地覆盖;总览同理。所有写入都是原子的,随后通过 `auto_tag_step` +更新这些文件的记忆标签 frontmatter;常规后台文件 watcher 会观察源文件变化并刷新派生索引。流程不会写入 JSONL、 中间 Markdown 或 Agent 结构化输出。 ## 参数 @@ -106,10 +108,13 @@ workspace 的 Markdown 目标。不存在、绝对路径、越界、带反斜杠 ## 产物 ```text -.reme/daily/YYYY-MM-DD/auto_fin.md +.reme/daily/YYYY-MM-DD/<主题笔记标题>.md # 每个有相关新闻的主题一份 +.reme/daily/YYYY-MM-DD/<当日总览标题>.md # 合并总览,回链到每一份主题笔记 ``` -报告包含标题、说明、当前 CLS 证据、历史分析、上下文 wikilink 和固定非投资建议声明。网络错误与无效 Agent 输出 +文件名取自 Agent 生成的中文标题,并经过非法字符净化和重名消歧。每份文件都带 `kind` frontmatter(`auto-fin-topic` 或 +`auto-fin-digest`),因此同日重跑会找回并覆盖自己产出的笔记,而不是重复生成。文件包含标题、说明、当前 CLS 证据、历史分析、上下文 +wikilink 和固定非投资建议声明;总览末尾另有一个 `## 主题详解` 列表,链接到各主题笔记。网络错误与无效 Agent 输出 会明确失败;没有相关当前新闻则成功跳过。 ## 验证 diff --git a/plugins/auto-fin/src/reme_auto_fin/__init__.py b/plugins/auto-fin/src/reme_auto_fin/__init__.py index 2c820d0d..63689d0c 100644 --- a/plugins/auto-fin/src/reme_auto_fin/__init__.py +++ b/plugins/auto-fin/src/reme_auto_fin/__init__.py @@ -1,14 +1,16 @@ """Auto Fin news research workflow.""" from .data import AutoFinDataStep -from .merge import AutoFinMergeStep -from .schema import AutoFinReportOutput, AutoFinTopicOutput +from .digest import AutoFinDigestStep +from .research import AutoFinResearchStep +from .schema import AutoFinNote, AutoFinReportOutput from .topic import AutoFinTopicStep __all__ = [ - "AutoFinReportOutput", "AutoFinDataStep", - "AutoFinMergeStep", - "AutoFinTopicOutput", + "AutoFinDigestStep", + "AutoFinNote", + "AutoFinReportOutput", + "AutoFinResearchStep", "AutoFinTopicStep", ] diff --git a/plugins/auto-fin/src/reme_auto_fin/base.py b/plugins/auto-fin/src/reme_auto_fin/base.py index eebea411..928f2c81 100644 --- a/plugins/auto-fin/src/reme_auto_fin/base.py +++ b/plugins/auto-fin/src/reme_auto_fin/base.py @@ -1,24 +1,42 @@ -"""Shared helpers for the Auto Fin workflow.""" +"""Shared Markdown, context, and Agent helpers for the Auto Fin workflow.""" from __future__ import annotations +from datetime import datetime, timezone from html.parser import HTMLParser import json import os from pathlib import Path +import re from time import perf_counter from typing import Any from uuid import uuid4 +import aiofiles +import frontmatter from pydantic import BaseModel from reme.steps import BaseStep +from reme.steps.file_io import get_path_lock, validate_filename_component + +from .schema import AutoFinReportOutput AGENT_INPUT_LOG_LIMIT = 2000 AGENT_OUTPUT_LOG_LIMIT = 4000 +NOTE_CHAR_LIMIT = 30_000 + +_WIKILINK = re.compile(r"\[\[([^\[\]\n]+)\]\]") +_HYBRID_WIKILINK = re.compile(r"(?P\[\[(?P[^\[\]\n]+)\]\])\((?P[^()\n]+)\)") +_HEADING = re.compile(r"^#+\s*") +_UNSAFE_FILENAME = re.compile(r'[<>:"/\\|?*\x00-\x1f]') + +FIRST_RUN_NOTICE = "今日暂无更早时段的推荐,本次为当日首次生成。" +DISCLAIMER = "> 未接入可靠行情数据;本文只提供新闻研究和回顾线索,不提供收益、目标价或买卖建议。" class _TextExtractor(HTMLParser): + """Flatten HTML into whitespace-separated text.""" + def __init__(self) -> None: super().__init__(convert_charrefs=True) self.parts: list[str] = [] @@ -41,25 +59,144 @@ class _TextExtractor(HTMLParser): self.parts.append(data) -def _plain_text(value: str) -> str: +def plain_text(value: str) -> str: + """Return one HTML fragment as collapsed plain text.""" parser = _TextExtractor() parser.feed(value) parser.close() return " ".join("".join(parser.parts).split()) -def _write(path: Path, text: str) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_name(f".{path.name}.{uuid4().hex}.tmp") +def utc_now_iso() -> str: + """Return the current UTC time as ISO-8601 for note frontmatter.""" + return datetime.now(timezone.utc).isoformat() + + +def normalize_title(raw: str, fallback: str) -> str: + """Return an Agent title as a safe Markdown filename stem.""" + title = _UNSAFE_FILENAME.sub("-", _HEADING.sub("", str(raw or "").strip())) + title = re.sub(r"\s+", " ", title).strip(" .-") + if title.lower().endswith(".md"): + title = title[:-3].strip(" .-") + title = title or fallback + if error := validate_filename_component(title, kind="title"): + raise ValueError(f"Unable to produce a safe Auto Fin title from {raw!r}: {error}") + return title + + +def normalize_report(output: AutoFinReportOutput) -> AutoFinReportOutput: + """Strip a duplicated H1 and fill in the fallbacks for one Agent report.""" + body = output.body.strip() + if body.startswith("# "): + body = body.partition("\n")[2].strip() + return output.model_copy( + update={ + "title": _HEADING.sub("", output.title.strip()) or "主题新闻观察", + "description": output.description.strip() or "基于当前新闻与历史记忆的主题研究。", + "body": body or "## 结论\n\n暂无可用结论。", + }, + ) + + +def normalize_hybrid_wikilinks(body: str) -> str: + """Drop a redundant Markdown destination from an unambiguous wikilink hybrid.""" + + def replace(match: re.Match[str]) -> str: + inner = match.group("inner").strip() + raw_target = inner.partition("|")[0].strip() + destination = match.group("destination").strip().removeprefix("<").removesuffix(">").strip() + return ( + match.group("wikilink") + if destination in {raw_target, raw_target.partition("#")[0].strip()} + else match.group(0) + ) + + return _HYBRID_WIKILINK.sub(replace, body) + + +def _valid_source_path(path: str) -> bool: + """Return whether one wikilink target could be a workspace-relative Markdown path.""" + return bool( + path + and not path.startswith("/") + and "\\" not in path + and path.endswith(".md") + and "." not in Path(path).parts + and ".." not in Path(path).parts + and not any(character in path for character in "[]|"), + ) + + +def validate_wikilinks(body: str, workspace: Path, exclude: Path) -> tuple[str, list[str]]: + """Keep real in-workspace Markdown links and downgrade invalid links to plain text.""" + workspace = workspace.resolve() + exclude = exclude.resolve() + sources: list[str] = [] + + def replace(match: re.Match[str]) -> str: + raw_target, separator, raw_alias = match.group(1).strip().partition("|") + path = raw_target.strip().partition("#")[0].strip() + alias = (raw_alias.strip() if separator else "") or Path(path).stem.replace("_", " ") + if not _valid_source_path(path): + return alias + resolved = (workspace / path).resolve() + if not resolved.is_relative_to(workspace) or not resolved.is_file() or resolved == exclude: + return alias + if path not in sources: + sources.append(path) + return match.group(0) + + return _WIKILINK.sub(replace, body), sources + + +def resolve_note_path(day_dir: Path, title: str, *, existing: Path | None) -> tuple[str, Path]: + """Return a title and path that neither reuse nor overwrite an unrelated note.""" + path = day_dir / f"{title}.md" + index = 2 + while path != existing and path.exists(): + path = day_dir / f"{title}({index}).md" + index += 1 + return path.stem, path + + +def find_note(day_dir: Path, *, kind: str, **matches: Any) -> Path | None: + """Return this workflow's note in one day directory whose frontmatter matches.""" + for path in sorted(day_dir.glob("*.md")) if day_dir.is_dir() else (): + try: + metadata = frontmatter.load(path).metadata + except (OSError, UnicodeError, ValueError): + continue + if metadata.get("kind") == kind and all(metadata.get(key) == value for key, value in matches.items()): + return path + return None + + +def read_note(path: Path | None) -> str: + """Return an earlier note's body for intra-day refinement, or the first-run notice.""" + if path is None: + return FIRST_RUN_NOTICE try: - temporary.write_text(text, encoding="utf-8") - os.replace(temporary, path) - finally: - temporary.unlink(missing_ok=True) + return frontmatter.load(path).content.strip()[:NOTE_CHAR_LIMIT] or FIRST_RUN_NOTICE + except (OSError, UnicodeError, ValueError): + return FIRST_RUN_NOTICE + + +async def write_markdown(path: Path, body: str, metadata: dict[str, Any]) -> None: + """Serialize one frontmatter Markdown document atomically under its path lock.""" + path.parent.mkdir(parents=True, exist_ok=True) + rendered = frontmatter.dumps(frontmatter.Post(body.strip(), **metadata)) + async with await get_path_lock(path): + temporary = path.with_name(f".{path.name}.{uuid4().hex}.tmp") + try: + async with aiofiles.open(temporary, "w", encoding="utf-8") as stream: + await stream.write(rendered if rendered.endswith("\n") else f"{rendered}\n") + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) class AutoFinStep(BaseStep): - """Shared Auto Fin helpers.""" + """Shared helpers for the steps in one Auto Fin RuntimeContext.""" def _value(self, key: str, default: Any = None) -> Any: assert self.context is not None @@ -71,6 +208,59 @@ class AutoFinStep(BaseStep): raise RuntimeError(f"Auto Fin data is missing: {key}") return value + @property + def day_dir(self) -> Path: + """Return the dated directory that holds this run's notes.""" + return self.workspace_path / str(self.config_value("daily_dir")) / str(self._required("auto_fin_date")) + + def _track(self, path: Path, existing: Path | None) -> None: + """Record this write, and any note it replaced, for the auto_tag step.""" + assert self.context is not None + changes = list(self.context.get("changes") or []) + if existing is not None and existing != path: + existing.unlink(missing_ok=True) + changes.append({"change": "deleted", "path": existing.relative_to(self.workspace_path).as_posix()}) + changes.append( + { + "change": "modified" if existing == path else "added", + "path": path.relative_to(self.workspace_path).as_posix(), + }, + ) + self.context["changes"] = changes + + async def _write_report( + self, + title: str, + description: str, + body: str, + *, + kind: str, + existing: Path | None = None, + trailer: str = "", + **metadata: Any, + ) -> tuple[str, str, list[str]]: + """Write one report note, returning its title, relative path, and valid sources.""" + title, path = resolve_note_path(self.day_dir, title, existing=existing) + body, sources = validate_wikilinks(normalize_hybrid_wikilinks(body), self.workspace_path, path) + await write_markdown( + path, + "\n\n".join(section for section in (body, trailer, DISCLAIMER) if section), + { + "name": title, + "title": title, + "description": description, + "kind": kind, + "generated_at": utc_now_iso(), + **metadata, + }, + ) + self._track(path, existing) + self.logger.info( + f"[{self.name}] wrote kind={kind} path={path.relative_to(self.workspace_path).as_posix()} " + f"chars={len(body)} sources={len(sources)}", + ) + return title, path.relative_to(self.workspace_path).as_posix(), sources + async def _reply( self, prompt_name: str, @@ -78,8 +268,9 @@ class AutoFinStep(BaseStep): job_tools: list[str] | None = None, injected_job_kwargs: dict[str, Any] | None = None, tool_context_id: str | None = None, - **values: str, + **values: Any, ) -> BaseModel: + """Ask the Agent for one structured report and validate it against ``model``.""" if self.agent_wrapper is None: raise RuntimeError("Auto Fin analysis requires an agent_wrapper") prompt = self.prompt_format(prompt_name, **values) @@ -93,9 +284,8 @@ class AutoFinStep(BaseStep): "builtin_tools": [], "use_builtin_tools": False, "skills": [], + "job_tools": list(self.kwargs.get("job_tools") or []) if job_tools is None else job_tools, } - if job_tools: - kwargs["job_tools"] = job_tools if injected_job_kwargs: kwargs["injected_job_kwargs"] = injected_job_kwargs if tool_context_id: @@ -105,10 +295,10 @@ class AutoFinStep(BaseStep): raise ValueError(f"Auto Fin Agent returned no structured output: {self._preview(result)}") value = result["structured_output"] output = value if isinstance(value, model) else model.model_validate(value) - output_preview = self._preview(output.model_dump(), AGENT_OUTPUT_LOG_LIMIT) + rendered = self._preview(output.model_dump(), AGENT_OUTPUT_LOG_LIMIT) self.logger.info( f"[{self.name}] agent output prompt={prompt_name} schema={model.__name__} " - f"elapsed={perf_counter() - started_at:.2f}s output={output_preview}", + f"elapsed={perf_counter() - started_at:.2f}s output={rendered}", ) return output diff --git a/plugins/auto-fin/src/reme_auto_fin/data.py b/plugins/auto-fin/src/reme_auto_fin/data.py index 127ab601..b061750a 100644 --- a/plugins/auto-fin/src/reme_auto_fin/data.py +++ b/plugins/auto-fin/src/reme_auto_fin/data.py @@ -9,7 +9,7 @@ from typing import Any import httpx -from .base import AutoFinStep, _plain_text +from .base import AutoFinStep, plain_text API_URL = "https://www.cls.cn/v1/roll/get_roll_list" HEADERS = { @@ -126,8 +126,8 @@ class AutoFinDataStep(AutoFinStep): return None if not start <= published_at <= end: return None - content = _plain_text(str(row.get("content") or row.get("brief") or "")) - title = _plain_text(str(row.get("title") or row.get("brief") or content)) + content = plain_text(str(row.get("content") or row.get("brief") or "")) + title = plain_text(str(row.get("title") or row.get("brief") or content)) if not title and not content: return None return { diff --git a/plugins/auto-fin/src/reme_auto_fin/digest.py b/plugins/auto-fin/src/reme_auto_fin/digest.py new file mode 100644 index 00000000..2971461f --- /dev/null +++ b/plugins/auto-fin/src/reme_auto_fin/digest.py @@ -0,0 +1,62 @@ +"""Merge the topic notes into one dated brief that links back to each of them.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +from reme.steps.file_io import refresh_day_index + +from .base import AutoFinStep, find_note, normalize_report, normalize_title, read_note +from .schema import AutoFinNote, AutoFinReportOutput + + +class AutoFinDigestStep(AutoFinStep): + """Summarize the topic notes and index the day once every note is in place.""" + + async def execute(self): + assert self.context is not None + if self.context.get("auto_fin_skipped"): + return self.context.response + notes: list[AutoFinNote] = self._required("auto_fin_notes") + run_date = str(self._required("auto_fin_date")) + earlier = find_note(self.day_dir, kind="auto-fin-digest") + output = normalize_report( + await self._reply( + "digest_user", + AutoFinReportOutput, + job_tools=[], + decision_at=str(self._required("auto_fin_decision_at")), + window_start=str(self._required("auto_fin_window_start")), + notes=json.dumps([note.model_dump() for note in notes], ensure_ascii=False), + earlier_brief=read_note(earlier), + ), + ) + title, path, sources = await self._write_report( + normalize_title(output.title, f"主题新闻观察({run_date})"), + output.description, + output.body, + kind="auto-fin-digest", + existing=earlier, + trailer="## 主题详解\n\n" + "\n".join(f"- [[{note.path}]]" for note in notes), + date=run_date, + source_notes=[note.path for note in notes], + ) + self.logger.info(f"[{self.name}] digest title={title!r} path={path}") + await refresh_day_index( + SimpleNamespace(workspace_path=self.workspace_path), + run_date, + str(self.config_value("daily_dir")), + ) + self.context["markdown_path"] = self.context["auto_fin_digest_path"] = path + self.context.response.answer = output.body + self.context.response.metadata.update( + { + "markdown_path": path, + "digest_path": path, + "source_paths": sources, + "note_paths": [note.path for note in notes], + "selected_news_count": len(self._required("auto_fin_selected_news")), + }, + ) + return self.context.response diff --git a/plugins/auto-fin/src/reme_auto_fin/digest.yaml b/plugins/auto-fin/src/reme_auto_fin/digest.yaml new file mode 100644 index 00000000..bbc39b4e --- /dev/null +++ b/plugins/auto-fin/src/reme_auto_fin/digest.yaml @@ -0,0 +1,20 @@ +digest_user: | + ## 输入材料 + 研究窗口:{window_start} 至 {decision_at} + 各主题笔记(JSON 列表,含 topic 与完整 Markdown body): + {notes} + 今天早些时段的总览: + {earlier_brief} + + ## 任务指令 + 你是当日总览 Agent。各主题笔记已经完成,你只负责把它们合成一篇总览,不再重新研究新闻,也没有任何搜索工具。 + + 写作要求: + - 用 `## 标题` 分层:先给跨主题的共同线索,再逐主题提炼要点。 + - 同一事件被多个主题收录时只写一次,并说明它同时影响哪些主题。 + - 只使用上面笔记中的事实,不得虚构行情、收益、价格或未提供的数据;证据不足处标明待核实。 + - 复核今天早些时段总览中仍然成立的判断;与笔记冲突时以笔记为准。 + - 做提炼而不是摘抄,不要整段复制笔记原文,也不要给投资建议。 + - 不要输出 wikilink,代码会在正文末尾自动附上各主题笔记的链接。 + + 按结构化输出契约返回 title、description 和完整中文 Markdown body。 \ No newline at end of file diff --git a/plugins/auto-fin/src/reme_auto_fin/merge.py b/plugins/auto-fin/src/reme_auto_fin/merge.py deleted file mode 100644 index 1aebc72c..00000000 --- a/plugins/auto-fin/src/reme_auto_fin/merge.py +++ /dev/null @@ -1,190 +0,0 @@ -"""Research current news with ReMe and save a wikilink-backed report.""" - -from __future__ import annotations - -import json -import re -from datetime import date, timedelta -from pathlib import Path -from types import SimpleNamespace -from uuid import uuid4 - -from reme.steps.file_io import refresh_day_index - -from .base import AutoFinStep, _write -from .schema import AutoFinReportOutput - -_WIKILINK_RE = re.compile(r"\[\[([^\[\]\n]+)\]\]") -_HYBRID_WIKILINK_RE = re.compile( - r"(?P\[\[(?P[^\[\]\n]+)\]\])\((?P[^()\n]+)\)", -) - - -class AutoFinMergeStep(AutoFinStep): - """Give one Agent read-only ReMe tools, then validate links in its Markdown.""" - - def _report_path(self, run_date: date) -> Path: - return self.workspace_path / str(self.config_value("daily_dir")) / str(run_date) / "auto_fin.md" - - def _current_report(self, run_date: date) -> str: - """Return today's existing report so intra-day reruns refine it, not replace it.""" - path = self._report_path(run_date) - if path.is_file(): - return path.read_text(encoding="utf-8")[:30_000] - return "今日暂无更早时段的推荐,本次为当日首次生成。" - - def _normalize_hybrid_wikilinks(self, body: str) -> str: - """Remove a redundant Markdown destination from an unambiguous wikilink hybrid.""" - - def replace(match: re.Match[str]) -> str: - inner = match.group("inner").strip() - raw_target = inner.partition("|")[0].strip() - target_path = raw_target.partition("#")[0].strip() - destination = match.group("destination").strip() - if destination.startswith("<") and destination.endswith(">"): - destination = destination[1:-1].strip() - if destination in {raw_target, target_path}: - return match.group("wikilink") - return match.group(0) - - try: - return _HYBRID_WIKILINK_RE.sub(replace, body) - except Exception as exc: # Defensive boundary: report generation must not depend on cosmetic normalization. - self.logger.warning(f"[{self.name}] failed to normalize hybrid wikilinks; keeping original body: {exc}") - return body - - @staticmethod - def _normalize(output: AutoFinReportOutput) -> AutoFinReportOutput: - title = re.sub(r"^#+\s*", "", output.title.strip()) or "主题新闻观察" - description = output.description.strip() or "基于当前新闻与历史记忆的主题研究。" - body = output.body.strip() or "## 结论\n\n暂无可用结论。" - if body.startswith("# "): - body = body.partition("\n")[2].lstrip() or "## 结论\n\n暂无可用结论。" - return output.model_copy(update={"title": title, "description": description, "body": body}) - - def _validate_wikilinks(self, body: str, run_date: date) -> tuple[str, list[str]]: - """Keep real in-workspace Markdown links and downgrade invalid links to text.""" - source_paths: list[str] = [] - report = self._report_path(run_date).resolve() - workspace = self.workspace_path.resolve() - - def replace(match: re.Match[str]) -> str: - inner = match.group(1).strip() - raw_target, separator, raw_alias = inner.partition("|") - target = raw_target.strip() - path = target.partition("#")[0].strip() - alias = (raw_alias.strip() if separator else "") or Path(path).stem.replace("_", " ") - if not self._valid_wikilink_path(path): - return alias - resolved = (workspace / path).resolve() - try: - resolved.relative_to(workspace) - except ValueError: - return alias - if not resolved.is_file() or resolved == report: - return alias - if path not in source_paths: - source_paths.append(path) - return match.group(0) - - return _WIKILINK_RE.sub(replace, body), source_paths - - @staticmethod - def _valid_wikilink_path(path: str) -> bool: - parts = Path(path).parts - return bool( - path - and not path.startswith("/") - and "\\" not in path - and path.endswith(".md") - and "." not in parts - and ".." not in parts - and not any(character in path for character in "[]|"), - ) - - async def execute(self): - """Research the selected news and persist the validated report.""" - assert self.context is not None - self.context["changes"] = [] - if self.context.get("auto_fin_skipped"): - return self.context.response - run_date = date.fromisoformat(str(self._required("auto_fin_date"))) - historical_search = { - "limit": 5, - "min_score": 0.0, - "start_date": None, - "end_date": (run_date - timedelta(days=1)).isoformat(), - "max_search_calls": 3, - } - by_topic = self._required("auto_fin_news_by_topic") - sections: list[str] = [] - source_paths: list[str] = [] - researched_topics: list[str] = [] - earlier_report = self._current_report(run_date) - for topic in self._required("auto_fin_topics"): - related = by_topic[topic] - if not related: - self.logger.info(f"[{self.name}] skipping topic={topic} reason=no_related_news") - continue - recent = sorted(related, key=lambda row: (row["event_time"], row["news_id"]), reverse=True)[:20] - tool_context_id = f"auto_fin:{uuid4().hex}" - self.logger.info( - f"[{self.name}] researching topic={topic} related_news={len(related)} " - f"sent_news={len(recent)} omitted_news={len(related) - len(recent)} search_limit=3", - ) - try: - output = await self._reply( - "merge_user", - AutoFinReportOutput, - job_tools=["search"], - injected_job_kwargs=historical_search, - tool_context_id=tool_context_id, - decision_at=str(self._required("auto_fin_decision_at")), - window_start=str(self._required("auto_fin_window_start")), - topic=topic, - news=json.dumps(recent, ensure_ascii=False), - omitted_news_count=str(len(related) - len(recent)), - current_report=earlier_report, - ) - finally: - if self.app_context is not None: - self.app_context.metadata.get("__search_call_budgets", {}).pop(tool_context_id, None) - output = self._normalize(output) - body, paths = self._validate_wikilinks(self._normalize_hybrid_wikilinks(output.body), run_date) - source_paths.extend(path for path in paths if path not in source_paths) - sections.append(f"## {topic}:{output.title}\n\n> {output.description}\n\n{body}") - researched_topics.append(topic) - self.logger.info( - f"[{self.name}] researched topic={topic} body_chars={len(body)} valid_sources={len(paths)}", - ) - title = f"主题新闻观察({run_date})" - description = f"截至 {self._required('auto_fin_decision_at')},关注 {', '.join(researched_topics)}。" - body = "\n\n".join(sections) - markdown = f"# {title}\n\n> {description}\n\n{body}\n\n" - markdown += "> 未接入可靠行情数据;本文只提供新闻研究和回顾线索,不提供收益、目标价或买卖建议。\n" - report = self._report_path(run_date) - change = "modified" if report.is_file() else "added" - _write(report, markdown) - self.logger.info( - f"[{self.name}] wrote report path={report.relative_to(self.workspace_path)} " - f"change={change} topics={len(researched_topics)} sources={len(source_paths)} chars={len(markdown)}", - ) - await refresh_day_index( - SimpleNamespace(workspace_path=self.workspace_path), - str(run_date), - str(self.config_value("daily_dir")), - ) - relative = report.relative_to(self.workspace_path).as_posix() - self.context["changes"] = [{"change": change, "path": relative}] - self.context["markdown_path"] = relative - self.context["auto_fin_digest_path"] = relative - self.context.response.answer = body - self.context.response.metadata.update( - { - "markdown_path": relative, - "digest_path": relative, - "source_paths": source_paths, - "selected_news_count": len(self._required("auto_fin_selected_news")), - }, - ) - return self.context.response diff --git a/plugins/auto-fin/src/reme_auto_fin/plugin.yaml b/plugins/auto-fin/src/reme_auto_fin/plugin.yaml index a9464e56..903b4d60 100644 --- a/plugins/auto-fin/src/reme_auto_fin/plugin.yaml +++ b/plugins/auto-fin/src/reme_auto_fin/plugin.yaml @@ -1,7 +1,8 @@ backends: auto_fin_data_step: reme_auto_fin.data:AutoFinDataStep auto_fin_topic_step: reme_auto_fin.topic:AutoFinTopicStep - auto_fin_merge_step: reme_auto_fin.merge:AutoFinMergeStep + auto_fin_research_step: reme_auto_fin.research:AutoFinResearchStep + auto_fin_digest_step: reme_auto_fin.digest:AutoFinDigestStep application_defaults: jobs: @@ -41,8 +42,9 @@ application_defaults: steps: &auto_fin_steps - backend: auto_fin_data_step - backend: auto_fin_topic_step - - backend: auto_fin_merge_step + - backend: auto_fin_research_step job_tools: [search] + - backend: auto_fin_digest_step - backend: auto_tag_step auto_fin_cron: diff --git a/plugins/auto-fin/src/reme_auto_fin/research.py b/plugins/auto-fin/src/reme_auto_fin/research.py new file mode 100644 index 00000000..41b85e24 --- /dev/null +++ b/plugins/auto-fin/src/reme_auto_fin/research.py @@ -0,0 +1,93 @@ +"""Research each topic's current news and persist one note per topic.""" + +from __future__ import annotations + +import json +from datetime import date, timedelta +from typing import Any +from uuid import uuid4 + +from .base import AutoFinStep, find_note, normalize_report, normalize_title, read_note +from .schema import AutoFinNote, AutoFinReportOutput + +NEWS_LIMIT = 20 +SEARCH_LIMIT = 3 + + +class AutoFinResearchStep(AutoFinStep): + """Research every topic independently so each one earns its own note.""" + + @staticmethod + def _search_budget(run_date: str) -> dict[str, Any]: + """Return the read-only historical search budget granted to one topic.""" + return { + "limit": 5, + "min_score": 0.0, + "start_date": None, + "end_date": (date.fromisoformat(run_date) - timedelta(days=1)).isoformat(), + "max_search_calls": SEARCH_LIMIT, + } + + async def _research(self, topic: str, related: list[dict], run_date: str) -> AutoFinNote: + """Research one topic and write its validated note.""" + recent = sorted(related, key=lambda row: (row["event_time"], row["news_id"]), reverse=True)[:NEWS_LIMIT] + earlier = find_note(self.day_dir, kind="auto-fin-topic", topic=topic) + context_id = f"auto_fin:{uuid4().hex}" + self.logger.info( + f"[{self.name}] research topic={topic} related={len(related)} " + f"sent={len(recent)} omitted={len(related) - len(recent)}", + ) + try: + output = normalize_report( + await self._reply( + "research_user", + AutoFinReportOutput, + job_tools=["search"], + injected_job_kwargs=self._search_budget(run_date), + tool_context_id=context_id, + decision_at=str(self._required("auto_fin_decision_at")), + window_start=str(self._required("auto_fin_window_start")), + topic=topic, + news=json.dumps(recent, ensure_ascii=False), + omitted_news_count=str(len(related) - len(recent)), + earlier_note=read_note(earlier), + ), + ) + finally: + if self.app_context is not None: + self.app_context.metadata.get("__search_call_budgets", {}).pop(context_id, None) + title, path, _ = await self._write_report( + normalize_title(output.title, f"{topic}观察"), + output.description, + output.body, + kind="auto-fin-topic", + existing=earlier, + date=run_date, + topic=topic, + source_news_ids=[row["news_id"] for row in recent], + ) + return AutoFinNote(topic=topic, title=title, description=output.description, body=output.body, path=path) + + async def execute(self): + """Write one note per topic that had relevant news.""" + assert self.context is not None + self.context["changes"] = [] + if self.context.get("auto_fin_skipped"): + return self.context.response + run_date = str(self._required("auto_fin_date")) + by_topic = self._required("auto_fin_news_by_topic") + notes: list[AutoFinNote] = [] + for topic in self._required("auto_fin_topics"): + if not by_topic[topic]: + self.logger.info(f"[{self.name}] skipping topic={topic} reason=no_related_news") + continue + notes.append(await self._research(topic, by_topic[topic], run_date)) + self.context["auto_fin_notes"] = notes + self.context.response.answer = f"Researched {len(notes)} topic(s): {', '.join(note.path for note in notes)}" + self.context.response.metadata.update( + { + "note_paths": [note.path for note in notes], + "selected_news_count": len(self._required("auto_fin_selected_news")), + }, + ) + return self.context.response diff --git a/plugins/auto-fin/src/reme_auto_fin/merge.yaml b/plugins/auto-fin/src/reme_auto_fin/research.yaml similarity index 61% rename from plugins/auto-fin/src/reme_auto_fin/merge.yaml rename to plugins/auto-fin/src/reme_auto_fin/research.yaml index 4ec8297e..125bec0e 100644 --- a/plugins/auto-fin/src/reme_auto_fin/merge.yaml +++ b/plugins/auto-fin/src/reme_auto_fin/research.yaml @@ -1,23 +1,25 @@ -merge_user: | +research_user: | ## 输入材料 研究窗口:{window_start} 至 {decision_at} 当前主题:{topic} 该主题最新的新闻(最多 20 篇,另有 {omitted_news_count} 篇未送入本次研究): {news} - 今天早些时段的报告: - {current_report} + 今天早些时段该主题的笔记: + {earlier_note} ## 任务指令 - 你是主题新闻研究 Agent。当前新闻已按主题筛选。你只能使用 `search` 搜索历史记忆,最多调用 3 次。 - 不得使用外部搜索,不得虚构行情、收益、价格或未提供的数据。搜索片段不足以支持的细节应标明待核实。 - 复核今天早些时段报告中与当前主题有关的判断。 - 围绕当前主题和重要事件检索历史记忆;用完搜索额度后直接完成报告。 + 你是单主题新闻研究 Agent。当前新闻已按主题筛选,你只研究「{topic}」这一个主题。 + 你只能使用 `search` 搜索历史记忆,最多调用 3 次。不得使用外部搜索,不得虚构行情、收益、价格或未提供的数据。 + 搜索片段不足以支持的细节应标明待核实。 + 复核今天早些时段笔记中与当前主题有关的判断;本次证据与之冲突时,说明冲突点和哪一方更可信。 + + 围绕当前主题和重要事件检索历史记忆;用完搜索额度后直接完成笔记。 说明历史事件与当前事件的相同点、关键差异,以及旧判断是否仍适用。 给出值得回顾的新闻、应继续观察的信息,以及哪些条件会强化或推翻判断,但不要给出投资建议。 当前新闻不是 workspace 文件:在正文中保留其 CLS news_id、发布时间和标题,不要为它虚构 wikilink。 对搜索结果中实际支持判断的历史 Markdown,把完整 workspace-relative path 作为 contextual wikilink 写进 相关句子,例如:这与[[daily/2026-08-01/auto_fin.md|此前的供给判断]]相似,但本次政策范围不同。 - 不得写裸链接行或虚构路径。代码会把不存在、越界或指向当前报告的链接降级为普通文本。 + 不得写裸链接行或虚构路径。代码会把不存在、越界或指向本文的链接降级为普通文本。 - 按结构化输出契约返回 title、description 和完整中文 Markdown body。 + 按结构化输出契约返回 title、description 和完整中文 Markdown body。 \ No newline at end of file diff --git a/plugins/auto-fin/src/reme_auto_fin/schema.py b/plugins/auto-fin/src/reme_auto_fin/schema.py index 3db10c5b..3e5db0f8 100644 --- a/plugins/auto-fin/src/reme_auto_fin/schema.py +++ b/plugins/auto-fin/src/reme_auto_fin/schema.py @@ -18,14 +18,18 @@ class AutoFinAgentModel(AutoFinModel): class AutoFinReportOutput(AutoFinAgentModel): - """Final Markdown returned by the agentic news-research Agent.""" + """One Chinese Markdown report returned by an Auto Fin Agent.""" title: str description: str body: str -class AutoFinTopicOutput(AutoFinAgentModel): - """CLS news identifiers that are semantically related to configured topics.""" +class AutoFinNote(AutoFinModel): + """One topic note the research step hands to the digest step.""" - news_ids: list[str] + topic: str + title: str + description: str + body: str + path: str diff --git a/plugins/auto-fin/tests/test_auto_fin.py b/plugins/auto-fin/tests/test_auto_fin.py index 55823620..0dfa564d 100644 --- a/plugins/auto-fin/tests/test_auto_fin.py +++ b/plugins/auto-fin/tests/test_auto_fin.py @@ -3,6 +3,7 @@ # pylint: disable=missing-function-docstring,protected-access import json +import re from datetime import datetime from pathlib import Path from zoneinfo import ZoneInfo @@ -10,10 +11,11 @@ from zoneinfo import ZoneInfo import pytest import yaml -from reme_auto_fin.base import _plain_text, _write +from reme_auto_fin.base import normalize_hybrid_wikilinks, normalize_title, plain_text, write_markdown from reme_auto_fin.data import AutoFinDataStep -from reme_auto_fin.merge import AutoFinMergeStep -from reme_auto_fin.schema import AutoFinReportOutput, AutoFinTopicOutput +from reme_auto_fin.digest import AutoFinDigestStep +from reme_auto_fin.research import AutoFinResearchStep +from reme_auto_fin.schema import AutoFinNote, AutoFinReportOutput from reme_auto_fin.topic import AutoFinTopicStep from reme.components import ApplicationContext from reme.components.agent_wrapper.base_agent_wrapper import BaseAgentWrapper @@ -23,6 +25,13 @@ SHANGHAI = ZoneInfo("Asia/Shanghai") PLUGIN_MANIFEST = yaml.safe_load( (Path(__file__).parents[1] / "src" / "reme_auto_fin" / "plugin.yaml").read_text(encoding="utf-8"), ) +LINKED_BODY = ( + "## 今日判断\n\n" + "CLS 1(09:00,黄金上涨)与 " + "[[daily/2026-08-01/auto_fin.md|历史黄金观察]]" + "(daily/2026-08-01/auto_fin.md) 背景相似。\n\n" + "无效引用 [[daily/missing.md|缺失文章]] 和 [[../../outside.md|越界文章]] 应降级。" +) def _row(news_id: int, value: datetime, title: str = "新闻", content: str = "正文") -> dict: @@ -34,7 +43,29 @@ def _row(news_id: int, value: datetime, title: str = "新闻", content: str = " } -def test_atomic_write_preserves_existing_file_on_failure(tmp_path: Path, monkeypatch): +def _news(news_id: str, event_time: str, title: str = "新闻") -> dict: + return {"news_id": news_id, "event_time": event_time, "title": title, "content": "正文"} + + +def _context(**kwargs) -> RuntimeContext: + defaults = { + "auto_fin_date": "2026-08-10", + "auto_fin_decision_at": "2026-08-10T09:30:00+08:00", + "auto_fin_window_start": "2026-08-09T09:30:00+08:00", + "auto_fin_topics": ["黄金", "机器人", "半导体"], + "auto_fin_selected_news": [_news("1", "2026-08-10T09:00:00+08:00")], + } + return RuntimeContext(**{**defaults, **kwargs}) + + +def _history(tmp_path: Path) -> None: + historical = tmp_path / "daily" / "2026-08-01" / "auto_fin.md" + historical.parent.mkdir(parents=True, exist_ok=True) + historical.write_text("# 历史黄金观察\n", encoding="utf-8") + + +@pytest.mark.asyncio +async def test_write_markdown_preserves_existing_file_on_failure(tmp_path: Path, monkeypatch): path = tmp_path / "result.md" path.write_text("existing", encoding="utf-8") monkeypatch.setattr( @@ -43,11 +74,14 @@ def test_atomic_write_preserves_existing_file_on_failure(tmp_path: Path, monkeyp ) with pytest.raises(OSError): - _write(path, "replacement") + await write_markdown(path, "replacement", {"name": "标题"}) assert path.read_text(encoding="utf-8") == "existing" assert not list(tmp_path.glob(".*.tmp")) - assert _plain_text("

甲&乙

丙

") == "甲&乙 丙" + + +def test_plain_text_drops_hidden_and_unescapes_entities(): + assert plain_text("

甲&乙

丙

") == "甲&乙 丙" @pytest.mark.asyncio @@ -238,157 +272,188 @@ async def test_topic_step_batches_by_prompt_length_and_merges_topics(tmp_path: P assert response.metadata["topic_batch_count"] == 3 -class _ResearchAgent(BaseAgentWrapper): +class _ReportAgent(BaseAgentWrapper): + """Return one titled Markdown report per call, keyed off the research topic.""" + def __init__(self, **kwargs): super().__init__(**kwargs) self.calls = [] async def reply(self, inputs, **kwargs): - self.calls.append((str(inputs), kwargs)) + prompt = str(inputs) + self.calls.append((prompt, kwargs)) + topic = re.search(r"当前主题:(\S+)", prompt) + title = f"{topic.group(1)}观察" if topic else "主题新闻观察" return { "structured_output": AutoFinReportOutput( - title="# 主题新闻观察", - description="关注黄金政策变化。", - body=( - "## 今日判断\n\n" - "CLS 1(09:00,黄金上涨)与 " - "[[daily/2026-08-01/auto_fin.md|历史黄金观察]]" - "(daily/2026-08-01/auto_fin.md) 背景相似。\n\n" - "无效引用 [[daily/missing.md|缺失文章]] 和 [[../../outside.md|越界文章]] 应降级。" - ), + title=f"# {title}", + description="关注政策变化。", + body=LINKED_BODY, ), } @pytest.mark.asyncio -async def test_merge_writes_only_final_report_and_validates_historical_links( - tmp_path: Path, -): - historical = tmp_path / "daily" / "2026-08-01" / "auto_fin.md" - historical.parent.mkdir(parents=True) - historical.write_text("# 历史黄金观察\n", encoding="utf-8") +async def test_research_writes_one_note_per_topic_with_latest_twenty_news(tmp_path: Path): + _history(tmp_path) app_context = ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai") - agent = _ResearchAgent(app_context=app_context) - context = RuntimeContext( - auto_fin_date="2026-08-10", - auto_fin_decision_at="2026-08-10T09:30:00+08:00", - auto_fin_window_start="2026-08-09T09:30:00+08:00", - auto_fin_topics=["黄金"], - auto_fin_news_by_topic={ - "黄金": [ - { - "news_id": "1", - "event_time": "2026-08-10T09:00:00+08:00", - "title": "黄金上涨", - "content": "避险需求增强", - }, - ], - }, - auto_fin_selected_news=[ - { - "news_id": "1", - "event_time": "2026-08-10T09:00:00+08:00", - "title": "黄金上涨", - "content": "避险需求增强", - }, - ], + agent = _ReportAgent(app_context=app_context) + gold = [_news(str(index), f"2026-08-10T09:{index:02}:00+08:00", "黄金") for index in range(25)] + context = _context( + auto_fin_news_by_topic={"黄金": gold, "机器人": [_news("30", "2026-08-10T08:00:00+08:00")], "半导体": []}, + auto_fin_selected_news=[*gold, _news("30", "2026-08-10T08:00:00+08:00")], ) - response = await AutoFinMergeStep( - app_context=app_context, - agent_wrapper=agent, - job_tools=["search"], - )(context) + response = await AutoFinResearchStep(app_context=app_context, agent_wrapper=agent)(context) - prompt, kwargs = agent.calls[0] - assert "end_date" not in prompt - assert "`search`" in prompt - assert "`read`" not in prompt - assert prompt.index("## 输入材料") < prompt.index("## 任务指令") - assert kwargs["output_schema"] == AutoFinReportOutput - assert kwargs["job_tools"] == ["search"] - assert kwargs["tool_context_id"].startswith("auto_fin:") - assert kwargs["injected_job_kwargs"] == { + assert len(agent.calls) == 2 + gold_prompt, gold_kwargs = agent.calls[0] + assert '"news_id": "24"' in gold_prompt + assert '"news_id": "4"' not in gold_prompt + assert "另有 5 篇" in gold_prompt + assert "当前主题:机器人" in agent.calls[1][0] + assert gold_kwargs["job_tools"] == ["search"] + assert gold_kwargs["output_schema"] == AutoFinReportOutput + assert gold_kwargs["tool_context_id"].startswith("auto_fin:") + assert gold_kwargs["tool_context_id"] != agent.calls[1][1]["tool_context_id"] + assert gold_kwargs["injected_job_kwargs"] == { "limit": 5, "min_score": 0.0, "start_date": None, "end_date": "2026-08-09", "max_search_calls": 3, } - report = (tmp_path / "daily" / "2026-08-10" / "auto_fin.md").read_text(encoding="utf-8") - assert "[[daily/2026-08-01/auto_fin.md|历史黄金观察]]" in report - assert "](daily/2026-08-01/auto_fin.md)" not in report - assert "缺失文章" in report and "越界文章" in report - assert "missing.md" not in report and "outside.md" not in report - assert not (tmp_path / "daily" / "2026-08-10" / "auto_fin_news.md").exists() - assert not (tmp_path / "resource").exists() - assert context["changes"] == [{"change": "added", "path": "daily/2026-08-10/auto_fin.md"}] - assert response.metadata["source_paths"] == ["daily/2026-08-01/auto_fin.md"] + + day = tmp_path / "daily" / "2026-08-10" + assert sorted(path.name for path in day.glob("*.md")) == ["机器人观察.md", "黄金观察.md"] + note = (day / "黄金观察.md").read_text(encoding="utf-8") + assert "kind: auto-fin-topic" in note and "topic: 黄金" in note + assert "[[daily/2026-08-01/auto_fin.md|历史黄金观察]]" in note + assert "](daily/2026-08-01/auto_fin.md)" not in note + assert "缺失文章" in note and "越界文章" in note + assert "missing.md" not in note and "outside.md" not in note + assert "不提供收益、目标价或买卖建议" in note + + assert [item.path for item in context["auto_fin_notes"]] == [ + "daily/2026-08-10/黄金观察.md", + "daily/2026-08-10/机器人观察.md", + ] + assert context["changes"] == [ + {"change": "added", "path": "daily/2026-08-10/黄金观察.md"}, + {"change": "added", "path": "daily/2026-08-10/机器人观察.md"}, + ] + assert response.metadata["selected_news_count"] == 26 + assert response.metadata["note_paths"] == [item.path for item in context["auto_fin_notes"]] @pytest.mark.asyncio -async def test_merge_researches_each_topic_with_latest_twenty_news(tmp_path: Path): - app_context = ApplicationContext(workspace_dir=str(tmp_path)) - agent = _ResearchAgent(app_context=app_context) - gold = [ - {"news_id": str(index), "event_time": f"2026-08-10T09:{index:02}:00+08:00", "title": "黄金", "content": "正文"} - for index in range(25) - ] - robot = [{"news_id": "30", "event_time": "2026-08-10T08:00:00+08:00", "title": "机器人", "content": "正文"}] - context = RuntimeContext( - auto_fin_date="2026-08-10", - auto_fin_decision_at="2026-08-10T09:30:00+08:00", - auto_fin_window_start="2026-08-09T09:30:00+08:00", - auto_fin_topics=["黄金", "机器人", "半导体"], - auto_fin_news_by_topic={"黄金": gold, "机器人": robot, "半导体": []}, - auto_fin_selected_news=[*gold, *robot], - ) +async def test_research_rerun_replaces_the_same_note_for_a_topic(tmp_path: Path): + app_context = ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai") + agent = _ReportAgent(app_context=app_context) + news = {"黄金": [_news("1", "2026-08-10T09:00:00+08:00")], "机器人": [], "半导体": []} + step = AutoFinResearchStep(app_context=app_context, agent_wrapper=agent) - response = await AutoFinMergeStep(app_context=app_context, agent_wrapper=agent)(context) + first = _context(auto_fin_news_by_topic=news) + await step(first) + second = _context(auto_fin_news_by_topic=news) + await step(second) assert len(agent.calls) == 2 - gold_prompt, gold_kwargs = agent.calls[0] - robot_prompt, robot_kwargs = agent.calls[1] - assert '"news_id": "24"' in gold_prompt - assert '"news_id": "4"' not in gold_prompt - assert "另有 5 篇" in gold_prompt - assert "当前主题:机器人" in robot_prompt - assert gold_kwargs["tool_context_id"] != robot_kwargs["tool_context_id"] - assert response.answer.count("## ") >= 2 - assert response.metadata["selected_news_count"] == 26 - report = (tmp_path / "daily" / "2026-08-10" / "auto_fin.md").read_text(encoding="utf-8") - assert "## 黄金:" in report and "## 机器人:" in report + assert "本次为当日首次生成。" in agent.calls[0][0] + assert "## 今日判断" in agent.calls[1][0] + assert second["changes"] == [{"change": "modified", "path": "daily/2026-08-10/黄金观察.md"}] + assert [path.name for path in (tmp_path / "daily" / "2026-08-10").glob("*.md")] == ["黄金观察.md"] -def test_hybrid_wikilink_normalization_is_conservative_and_failure_safe(tmp_path: Path, monkeypatch): - import reme_auto_fin.merge as merge_module +@pytest.mark.asyncio +async def test_research_skips_topics_without_news_and_honours_the_skip_flag(tmp_path: Path): + app_context = ApplicationContext(workspace_dir=str(tmp_path)) + agent = _ReportAgent(app_context=app_context) + context = _context(auto_fin_news_by_topic={"黄金": [], "机器人": [], "半导体": []}) - step = AutoFinMergeStep( - app_context=ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai"), + response = await AutoFinResearchStep(app_context=app_context, agent_wrapper=agent)(context) + + assert agent.calls == [] + assert context["auto_fin_notes"] == [] + assert response.metadata["note_paths"] == [] + assert not (tmp_path / "daily").exists() + + skipped = _context(auto_fin_skipped=True, auto_fin_news_by_topic={"黄金": []}) + await AutoFinResearchStep(app_context=app_context, agent_wrapper=agent)(skipped) + assert len(agent.calls) == 0 + + +@pytest.mark.asyncio +async def test_digest_merges_notes_and_links_back_to_each_of_them(tmp_path: Path): + _history(tmp_path) + app_context = ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai") + agent = _ReportAgent(app_context=app_context) + context = _context( + auto_fin_news_by_topic={"黄金": [_news("1", "2026-08-10T09:00:00+08:00")], "机器人": [], "半导体": []}, ) + await AutoFinResearchStep(app_context=app_context, agent_wrapper=agent)(context) + note_path = context["auto_fin_notes"][0].path + + response = await AutoFinDigestStep(app_context=app_context, agent_wrapper=agent)(context) + + prompt, kwargs = agent.calls[-1] + assert kwargs["job_tools"] == [] + assert '"topic": "黄金"' in prompt + assert "各主题笔记" in prompt + assert "当前主题:" not in prompt + assert "本次为当日首次生成。" in prompt + + digest = (tmp_path / "daily" / "2026-08-10" / "主题新闻观察.md").read_text(encoding="utf-8") + assert "kind: auto-fin-digest" in digest + assert "## 主题详解" in digest + assert f"- [[{note_path}]]" in digest + assert "[[daily/2026-08-01/auto_fin.md|历史黄金观察]]" in digest + assert "缺失文章" in digest and "missing.md" not in digest + assert digest.rstrip().endswith("不提供收益、目标价或买卖建议。") + + assert context["markdown_path"] == "daily/2026-08-10/主题新闻观察.md" + assert context["changes"][-1] == {"change": "added", "path": "daily/2026-08-10/主题新闻观察.md"} + assert response.metadata["digest_path"] == context["markdown_path"] + assert response.metadata["source_paths"] == ["daily/2026-08-01/auto_fin.md"] + assert response.metadata["note_paths"] == [note_path] + + +@pytest.mark.asyncio +async def test_digest_skips_when_the_run_was_already_skipped(tmp_path: Path): + app_context = ApplicationContext(workspace_dir=str(tmp_path)) + agent = _ReportAgent(app_context=app_context) + context = _context(auto_fin_skipped=True) + + await AutoFinDigestStep(app_context=app_context, agent_wrapper=agent)(context) + + assert agent.calls == [] + assert not (tmp_path / "daily").exists() + + +def test_normalize_hybrid_wikilinks_is_conservative(tmp_path: Path): body = ( "[[digest/wiki/gold.md]](digest/wiki/gold.md) " "[[digest/wiki/gold.md|黄金]]() " "[[digest/wiki/gold.md#L2|黄金]](digest/wiki/gold.md) " "[[digest/wiki/gold.md]](digest/wiki/other.md)" ) - assert step._normalize_hybrid_wikilinks(body) == ( + + assert normalize_hybrid_wikilinks(body) == ( "[[digest/wiki/gold.md]] " "[[digest/wiki/gold.md|黄金]] " "[[digest/wiki/gold.md#L2|黄金]] " "[[digest/wiki/gold.md]](digest/wiki/other.md)" ) - class _BrokenPattern: - @staticmethod - def sub(_replace, _body): - raise RuntimeError("normalization failed") - monkeypatch.setattr(merge_module, "_HYBRID_WIKILINK_RE", _BrokenPattern()) - assert step._normalize_hybrid_wikilinks(body) == body +def test_normalize_title_sanitizes_agent_titles_and_keeps_notes_importable(): + assert normalize_title("# 黄金/政策:观察", "黄金观察") == "黄金-政策:观察" + assert normalize_title(" ", "黄金观察") == "黄金观察" + assert normalize_title("解读.md", "黄金观察") == "解读" + assert AutoFinNote(topic="黄金", title="标题", description="说明", body="正文", path="a.md").topic == "黄金" -def test_plugin_config_has_default_topics_and_no_intermediate_index_step(): +def test_plugin_config_has_default_topics_and_two_report_steps(): jobs = PLUGIN_MANIFEST["application_defaults"]["jobs"] job = jobs["auto_fin"] assert job["parameters"]["properties"]["topics"]["default"] == "黄金,机器人,半导体" @@ -396,14 +461,13 @@ def test_plugin_config_has_default_topics_and_no_intermediate_index_step(): assert job["parameters"]["properties"]["request_interval"]["default"] == 10 assert job["parameters"]["properties"]["max_retries"]["default"] == 3 assert "news_file" not in job["parameters"]["properties"] - assert [step["backend"] for step in job["steps"]] == [ - "auto_fin_data_step", - "auto_fin_topic_step", - "auto_fin_merge_step", - "auto_tag_step", + assert job["steps"] == [ + {"backend": "auto_fin_data_step"}, + {"backend": "auto_fin_topic_step"}, + {"backend": "auto_fin_research_step", "job_tools": ["search"]}, + {"backend": "auto_fin_digest_step"}, + {"backend": "auto_tag_step"}, ] - assert job["steps"][2]["job_tools"] == ["search"] - assert job["steps"][3] == {"backend": "auto_tag_step"} assert jobs["auto_fin_cron"]["cron"] == "0 9 * * *" assert jobs["auto_fin_cron"]["steps"] == job["steps"] assert ( @@ -416,11 +480,8 @@ def test_plugin_config_has_default_topics_and_no_intermediate_index_step(): ) -def test_agent_schemas_are_small_and_required(): - topic = AutoFinTopicOutput.model_json_schema() +def test_report_schema_is_small_and_required(): report = AutoFinReportOutput.model_json_schema() - assert topic["required"] == ["news_ids"] - assert set(topic["properties"]) == {"news_ids"} assert report["required"] == ["title", "description", "body"] assert set(report["properties"]) == {"title", "description", "body"} diff --git a/reme/config/cookbook.yaml b/reme/config/cookbook.yaml index 0bcc8efb..201d66b0 100644 --- a/reme/config/cookbook.yaml +++ b/reme/config/cookbook.yaml @@ -122,10 +122,13 @@ jobs: steps: &auto_fin_steps - backend: auto_fin_data_step - backend: auto_fin_topic_step - - backend: auto_fin_merge_step - job_tools: [search, read] + - backend: auto_fin_research_step + job_tools: [search] + - backend: auto_fin_digest_step - backend: auto_tag_step - <<: *dingtalk_send_defaults + input_mapping: + auto_fin_digest_path: markdown_path title: ReMe Auto Fin auto_fin_cron: diff --git a/tests/unit/test_cookbook_config.py b/tests/unit/test_cookbook_config.py index c01c3d42..233ea4b8 100644 --- a/tests/unit/test_cookbook_config.py +++ b/tests/unit/test_cookbook_config.py @@ -124,11 +124,15 @@ def test_cookbook_appends_dingtalk_to_business_pipelines(monkeypatch): assert [step["backend"] for step in auto_fin_steps] == [ "auto_fin_data_step", "auto_fin_topic_step", - "auto_fin_merge_step", + "auto_fin_research_step", + "auto_fin_digest_step", "auto_tag_step", "dingtalk_markdown_send_step", ] assert jobs["auto_fin_cron"]["steps"] == auto_fin_steps + assert auto_fin_steps[-1]["input_mapping"] == { + "auto_fin_digest_path": "markdown_path", + } assert auto_fin_steps[-1]["title"] == "ReMe Auto Fin" assert [step["backend"] for step in daily_paper_steps] == [