diff --git a/plugins/daily_paper/README.md b/plugins/daily_paper/README.md index 7537bb6b..fa837635 100644 --- a/plugins/daily_paper/README.md +++ b/plugins/daily_paper/README.md @@ -13,7 +13,7 @@ their Job configuration under `application_defaults`. Enable the installed plugi ### 1. Install ReMe and Daily Paper ```bash -python -m pip install "reme-ai[core]>=0.4.1.12" +python -m pip install "reme-ai[core]>=0.4.2.0" reme plugins install plugins/daily_paper ``` diff --git a/plugins/daily_paper/README_ZH.md b/plugins/daily_paper/README_ZH.md index c01ffa47..58350574 100644 --- a/plugins/daily_paper/README_ZH.md +++ b/plugins/daily_paper/README_ZH.md @@ -11,7 +11,7 @@ Step backend，并在 `application_defaults` 下提供 Job 配置；通过 `plug ### 1. 安装 ReMe 和每日论文插件 ```bash -python -m pip install "reme-ai[core]>=0.4.1.12" +python -m pip install "reme-ai[core]>=0.4.2.0" reme plugins install plugins/daily_paper ``` diff --git a/plugins/daily_paper/pyproject.toml b/plugins/daily_paper/pyproject.toml index eb131365..16bf9437 100644 --- a/plugins/daily_paper/pyproject.toml +++ b/plugins/daily_paper/pyproject.toml @@ -8,7 +8,7 @@ license-files = ["LICENSE"] requires-python = ">=3.11" dependencies = [ "pypdf>=5.0.0", - "reme-ai>=0.4.1.12", + "reme-ai>=0.4.2.0", ] [project.entry-points."reme.plugins"] diff --git a/plugins/daily_paper/src/reme_daily_paper/analyze.py b/plugins/daily_paper/src/reme_daily_paper/analyze.py index ae9c1b25..a2ecaf33 100644 --- a/plugins/daily_paper/src/reme_daily_paper/analyze.py +++ b/plugins/daily_paper/src/reme_daily_paper/analyze.py @@ -9,6 +9,7 @@ from .arxiv import ArxivPdfClient from .base import ( PAPER_COUNT, DailyPaperStep, + analysis_summary_metadata, iter_note_metadata, markdown_output, normalize_chinese_title, @@ -227,15 +228,7 @@ class DailyPaperAnalyzeStep(DailyPaperStep): self.logger.warning(f"[{self.name}] paper failed arxiv_id={item.arxiv_id}: {exc}") else: analyses.append(analysis) - self.context.response.metadata.update( - { - "analysis_count": len(analyses), - "failure_count": len(failures), - "failed_papers": list(failures), - "note_paths": [paper.note_path for paper in analyses], - "pdf_paths": [paper.pdf_path for paper in analyses], - }, - ) + self.context.response.metadata.update(analysis_summary_metadata(analyses, failures)) self.context.response.success = bool(analyses) self.context.response.answer = ( f"Agent wrote {len(analyses)} detailed paper notes; {len(failures)} paper(s) failed" diff --git a/plugins/daily_paper/src/reme_daily_paper/base.py b/plugins/daily_paper/src/reme_daily_paper/base.py index 2dd3f87c..3301d153 100644 --- a/plugins/daily_paper/src/reme_daily_paper/base.py +++ b/plugins/daily_paper/src/reme_daily_paper/base.py @@ -1,23 +1,20 @@ """Shared state and file helpers for daily-paper steps.""" import datetime as dt -import os import re import zoneinfo from collections.abc import Iterator from contextlib import asynccontextmanager from pathlib import Path from typing import Any, TypeVar -from uuid import uuid4 -import aiofiles import frontmatter from pydantic import BaseModel from reme.steps import BaseStep -from reme.steps.file_io import get_path_lock, validate_filename_component +from reme.steps.file_io import get_path_lock, validate_filename_component, write_file_safe -from .schema import DailyPaperMarkdownOutput +from .schema import AnalyzedPaper, DailyPaperMarkdownOutput # Number of papers selected, analyzed, and digested each run. Shared across steps. PAPER_COUNT = 3 @@ -70,6 +67,20 @@ def markdown_output(result: dict[str, Any]) -> DailyPaperMarkdownOutput: return output.model_copy(update={"desc": desc, "body": body}) +def analysis_summary_metadata( + analyses: list[AnalyzedPaper], + failures: list[dict[str, str]], +) -> dict[str, Any]: + """Build shared progress metadata for analyze and standalone digest steps.""" + return { + "analysis_count": len(analyses), + "failure_count": len(failures), + "failed_papers": list(failures), + "note_paths": [item.note_path for item in analyses], + "pdf_paths": [item.pdf_path for item in analyses], + } + + def normalize_chinese_title(raw: str, fallback: str) -> str: """Return one safe Chinese title that can also be used as the filename stem.""" title = replace_surrogates(str(raw or "").strip()) @@ -157,19 +168,12 @@ async def reserve_unique_note_path( async def write_atomic(path: Path, content: str | bytes) -> None: - """Write through a sibling temporary file under the repository path lock.""" + """Write atomically under the path lock, cleaning surrogate text first.""" path.parent.mkdir(parents=True, exist_ok=True) lock = await get_path_lock(path) async with lock: - temp_path = path.with_name(f".{path.name}.{uuid4().hex}.tmp") - payload = replace_surrogates(content).encode("utf-8") if isinstance(content, str) else content - try: - async with aiofiles.open(temp_path, "wb") as stream: - await stream.write(payload) - os.replace(temp_path, path) - finally: - if temp_path.exists(): - temp_path.unlink() + payload = replace_surrogates(content) if isinstance(content, str) else content + await write_file_safe(path, payload) async def write_markdown(path: Path, body: str, metadata: dict[str, Any]) -> None: diff --git a/plugins/daily_paper/src/reme_daily_paper/collect.py b/plugins/daily_paper/src/reme_daily_paper/collect.py index 3f25abf3..4d150e29 100644 --- a/plugins/daily_paper/src/reme_daily_paper/collect.py +++ b/plugins/daily_paper/src/reme_daily_paper/collect.py @@ -4,8 +4,6 @@ import asyncio import datetime as dt from pathlib import Path -import frontmatter - from .arxiv import ARXIV_ID_PATTERN from .base import DailyPaperStep, iter_note_metadata, now from .huggingface_papers import HuggingFacePapersClient @@ -65,11 +63,11 @@ class DailyPaperCollectStep(DailyPaperStep): return found @staticmethod - def find_saved_digest(day_dir: Path) -> Path | None: - """Find a generated brief by frontmatter instead of by its model-generated title.""" + def find_saved_digest(day_dir: Path) -> tuple[Path, dict] | None: + """Find a generated brief and return its already-read frontmatter.""" for path, metadata in iter_note_metadata(day_dir): if metadata.get("kind") == "daily-paper-brief" or path.name == "daily-paper-brief.md": - return path + return path, metadata return None @staticmethod @@ -85,6 +83,7 @@ class DailyPaperCollectStep(DailyPaperStep): return PaperInfo.model_validate(values) async def execute(self): + """Load prior brief state or collect the current weekly and monthly papers.""" assert self.context is not None timezone = self.app_context.app_config.timezone if self.app_context is not None else None raw_date = str(self._value("date", "") or "").strip() @@ -95,16 +94,16 @@ class DailyPaperCollectStep(DailyPaperStep): daily_dir = str(self.config_value("daily_dir")).strip("/") force = bool(self._value("force", False)) self.logger.info(f"[{self.name}] start date={day} force={force}") - digest_path = self.find_saved_digest(self.workspace_path / daily_dir / day) - if digest_path is not None: + saved_digest = self.find_saved_digest(self.workspace_path / daily_dir / day) + if saved_digest is not None: + digest_path, saved_metadata = saved_digest digest_rel = digest_path.relative_to(self.workspace_path).as_posix() self._set_state("existing_digest_path", digest_rel) if not force: self._set_state("skip", True) self._set_state("digest_path", digest_rel) self.context.response.success = True - saved = frontmatter.load(digest_path) - partial = bool(saved.metadata.get("partial", False)) + partial = bool(saved_metadata.get("partial", False)) qualifier = "partial " if partial else "" self.context.response.answer = ( f"Skipped: {qualifier}daily paper brief already exists at {digest_rel}" @@ -116,7 +115,7 @@ class DailyPaperCollectStep(DailyPaperStep): "digest_path": digest_rel, "skipped": True, "partial": partial, - "failed_arxiv_ids": saved.metadata.get("failed_arxiv_ids", []), + "failed_arxiv_ids": saved_metadata.get("failed_arxiv_ids", []), }, ) self.logger.info(f"[{self.name}] skip existing digest path={digest_rel}") diff --git a/plugins/daily_paper/src/reme_daily_paper/digest.py b/plugins/daily_paper/src/reme_daily_paper/digest.py index d11e8927..51973ab1 100644 --- a/plugins/daily_paper/src/reme_daily_paper/digest.py +++ b/plugins/daily_paper/src/reme_daily_paper/digest.py @@ -11,6 +11,7 @@ from reme.steps.file_io import refresh_day_index from .base import ( PAPER_COUNT, DailyPaperStep, + analysis_summary_metadata, markdown_output, normalize_chinese_title, reserve_unique_note_path, @@ -113,7 +114,7 @@ class DailyPaperDigestStep(DailyPaperStep): ) self.logger.info(f"[{self.name}] start notes={len(analyses)}") - documents = [{"title": item.title, "desc": item.desc, "body": item.body} for item in analyses] + documents = [item.model_dump(include={"title", "desc", "body"}) for item in analyses] wikilinks = [f"[[{item.note_path}]]" for item in analyses] run_day = dt.date.fromisoformat(self._run_day()) daily_dir = str(self.config_value("daily_dir")).strip("/") @@ -198,17 +199,13 @@ class DailyPaperDigestStep(DailyPaperStep): self.context.response.answer = f"Generated {qualifier}daily paper brief: {digest_rel}" self.context.response.metadata.update( { + **analysis_summary_metadata(analyses, failures), "date": day, "partial": partial, - "analysis_count": len(analyses), - "failure_count": len(failures), - "failed_papers": list(failures), "week": self._state("week"), "month": self._state("month"), "selection_reasoning": [item.reasoning for item in analyses], "selected_arxiv_ids": selected_ids, - "note_paths": [item.note_path for item in analyses], - "pdf_paths": [item.pdf_path for item in analyses], "digest_path": digest_rel, "source_counts": self._state("source_counts"), "excluded_yesterday_count": len( diff --git a/tests/unit/test_package_versions.py b/tests/unit/test_package_versions.py index f1c34c4a..70ff73a8 100644 --- a/tests/unit/test_package_versions.py +++ b/tests/unit/test_package_versions.py @@ -224,7 +224,7 @@ def test_daily_paper_declares_runtime_dependencies() -> None: assert not by_name["reme-ai"].extras assert Version("0.4.1.11") not in by_name["reme-ai"].specifier - assert Version("0.4.1.12") in by_name["reme-ai"].specifier + assert Version("0.4.2.0") in by_name["reme-ai"].specifier assert "pypdf" in by_name