scriptoriumadmin/ReMe
1
0
Fork 0
mirror of https://github.com/agentscope-ai/ReMe.git synced 2026-11-14 09:01:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

refactor(daily-paper): reuse core persistence helpers

Browse source
This commit is contained in:
adarshh347 2026-11-03 19:04:06 +05:30
parent 85a9fa2607
commit dded3ee4b8
8 changed files with 37 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
plugins/daily_paper/README.md
View file

@ -13,7 +13,7 @@ their Job configuration under `application_defaults`. Enable the installed plugi
### 1. Install ReMe and Daily Paper
```bash
python -m pip install "reme-ai[core]>=0.4.1.12"
python -m pip install "reme-ai[core]>=0.4.2.0"
reme plugins install plugins/daily_paper
```

2
plugins/daily_paper/README_ZH.md
View file

@ -11,7 +11,7 @@ Step backend，并在 `application_defaults` 下提供 Job 配置；通过 `plug
### 1. 安装 ReMe 和每日论文插件
```bash
python -m pip install "reme-ai[core]>=0.4.1.12"
python -m pip install "reme-ai[core]>=0.4.2.0"
reme plugins install plugins/daily_paper
```

2
plugins/daily_paper/pyproject.toml
View file

@ -8,7 +8,7 @@ license-files = ["LICENSE"]
requires-python = ">=3.11"
dependencies = [
"pypdf>=5.0.0",
"reme-ai>=0.4.1.12",
"reme-ai>=0.4.2.0",
]
[project.entry-points."reme.plugins"]

11
plugins/daily_paper/src/reme_daily_paper/analyze.py
View file

@ -9,6 +9,7 @@ from .arxiv import ArxivPdfClient
from .base import (
PAPER_COUNT,
DailyPaperStep,
analysis_summary_metadata,
iter_note_metadata,
markdown_output,
normalize_chinese_title,
@ -227,15 +228,7 @@ class DailyPaperAnalyzeStep(DailyPaperStep):
self.logger.warning(f"[{self.name}] paper failed arxiv_id={item.arxiv_id}: {exc}")
else:
analyses.append(analysis)
self.context.response.metadata.update(
{
"analysis_count": len(analyses),
"failure_count": len(failures),
"failed_papers": list(failures),
"note_paths": [paper.note_path for paper in analyses],
"pdf_paths": [paper.pdf_path for paper in analyses],
},
)
self.context.response.metadata.update(analysis_summary_metadata(analyses, failures))
self.context.response.success = bool(analyses)
self.context.response.answer = (
f"Agent wrote {len(analyses)} detailed paper notes; {len(failures)} paper(s) failed"

34
plugins/daily_paper/src/reme_daily_paper/base.py
View file

@ -1,23 +1,20 @@
"""Shared state and file helpers for daily-paper steps."""
import datetime as dt
import os
import re
import zoneinfo
from collections.abc import Iterator
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any, TypeVar
from uuid import uuid4
import aiofiles
import frontmatter
from pydantic import BaseModel
from reme.steps import BaseStep
from reme.steps.file_io import get_path_lock, validate_filename_component
from reme.steps.file_io import get_path_lock, validate_filename_component, write_file_safe
from .schema import DailyPaperMarkdownOutput
from .schema import AnalyzedPaper, DailyPaperMarkdownOutput
# Number of papers selected, analyzed, and digested each run. Shared across steps.
PAPER_COUNT = 3
@ -70,6 +67,20 @@ def markdown_output(result: dict[str, Any]) -> DailyPaperMarkdownOutput:
return output.model_copy(update={"desc": desc, "body": body})
def analysis_summary_metadata(
analyses: list[AnalyzedPaper],
failures: list[dict[str, str]],
) -> dict[str, Any]:
"""Build shared progress metadata for analyze and standalone digest steps."""
return {
"analysis_count": len(analyses),
"failure_count": len(failures),
"failed_papers": list(failures),
"note_paths": [item.note_path for item in analyses],
"pdf_paths": [item.pdf_path for item in analyses],
}
def normalize_chinese_title(raw: str, fallback: str) -> str:
"""Return one safe Chinese title that can also be used as the filename stem."""
title = replace_surrogates(str(raw or "").strip())
@ -157,19 +168,12 @@ async def reserve_unique_note_path(
async def write_atomic(path: Path, content: str | bytes) -> None:
"""Write through a sibling temporary file under the repository path lock."""
"""Write atomically under the path lock, cleaning surrogate text first."""
path.parent.mkdir(parents=True, exist_ok=True)
lock = await get_path_lock(path)
async with lock:
temp_path = path.with_name(f".{path.name}.{uuid4().hex}.tmp")
payload = replace_surrogates(content).encode("utf-8") if isinstance(content, str) else content
try:
async with aiofiles.open(temp_path, "wb") as stream:
await stream.write(payload)
os.replace(temp_path, path)
finally:
if temp_path.exists():
temp_path.unlink()
payload = replace_surrogates(content) if isinstance(content, str) else content
await write_file_safe(path, payload)
async def write_markdown(path: Path, body: str, metadata: dict[str, Any]) -> None:

19
plugins/daily_paper/src/reme_daily_paper/collect.py
View file

@ -4,8 +4,6 @@ import asyncio
import datetime as dt
from pathlib import Path
import frontmatter
from .arxiv import ARXIV_ID_PATTERN
from .base import DailyPaperStep, iter_note_metadata, now
from .huggingface_papers import HuggingFacePapersClient
@ -65,11 +63,11 @@ class DailyPaperCollectStep(DailyPaperStep):
return found
@staticmethod
def find_saved_digest(day_dir: Path) -> Path | None:
"""Find a generated brief by frontmatter instead of by its model-generated title."""
def find_saved_digest(day_dir: Path) -> tuple[Path, dict] | None:
"""Find a generated brief and return its already-read frontmatter."""
for path, metadata in iter_note_metadata(day_dir):
if metadata.get("kind") == "daily-paper-brief" or path.name == "daily-paper-brief.md":
return path
return path, metadata
return None
@staticmethod
@ -85,6 +83,7 @@ class DailyPaperCollectStep(DailyPaperStep):
return PaperInfo.model_validate(values)
async def execute(self):
"""Load prior brief state or collect the current weekly and monthly papers."""
assert self.context is not None
timezone = self.app_context.app_config.timezone if self.app_context is not None else None
raw_date = str(self._value("date", "") or "").strip()
@ -95,16 +94,16 @@ class DailyPaperCollectStep(DailyPaperStep):
daily_dir = str(self.config_value("daily_dir")).strip("/")
force = bool(self._value("force", False))
self.logger.info(f"[{self.name}] start date={day} force={force}")
digest_path = self.find_saved_digest(self.workspace_path / daily_dir / day)
if digest_path is not None:
saved_digest = self.find_saved_digest(self.workspace_path / daily_dir / day)
if saved_digest is not None:
digest_path, saved_metadata = saved_digest
digest_rel = digest_path.relative_to(self.workspace_path).as_posix()
self._set_state("existing_digest_path", digest_rel)
if not force:
self._set_state("skip", True)
self._set_state("digest_path", digest_rel)
self.context.response.success = True
saved = frontmatter.load(digest_path)
partial = bool(saved.metadata.get("partial", False))
partial = bool(saved_metadata.get("partial", False))
qualifier = "partial " if partial else ""
self.context.response.answer = (
f"Skipped: {qualifier}daily paper brief already exists at {digest_rel}"
@ -116,7 +115,7 @@ class DailyPaperCollectStep(DailyPaperStep):
"digest_path": digest_rel,
"skipped": True,
"partial": partial,
"failed_arxiv_ids": saved.metadata.get("failed_arxiv_ids", []),
"failed_arxiv_ids": saved_metadata.get("failed_arxiv_ids", []),
},
)
self.logger.info(f"[{self.name}] skip existing digest path={digest_rel}")

9
plugins/daily_paper/src/reme_daily_paper/digest.py
View file

@ -11,6 +11,7 @@ from reme.steps.file_io import refresh_day_index
from .base import (
PAPER_COUNT,
DailyPaperStep,
analysis_summary_metadata,
markdown_output,
normalize_chinese_title,
reserve_unique_note_path,
@ -113,7 +114,7 @@ class DailyPaperDigestStep(DailyPaperStep):
)
self.logger.info(f"[{self.name}] start notes={len(analyses)}")
documents = [{"title": item.title, "desc": item.desc, "body": item.body} for item in analyses]
documents = [item.model_dump(include={"title", "desc", "body"}) for item in analyses]
wikilinks = [f"[[{item.note_path}]]" for item in analyses]
run_day = dt.date.fromisoformat(self._run_day())
daily_dir = str(self.config_value("daily_dir")).strip("/")
@ -198,17 +199,13 @@ class DailyPaperDigestStep(DailyPaperStep):
self.context.response.answer = f"Generated {qualifier}daily paper brief: {digest_rel}"
self.context.response.metadata.update(
{
**analysis_summary_metadata(analyses, failures),
"date": day,
"partial": partial,
"analysis_count": len(analyses),
"failure_count": len(failures),
"failed_papers": list(failures),
"week": self._state("week"),
"month": self._state("month"),
"selection_reasoning": [item.reasoning for item in analyses],
"selected_arxiv_ids": selected_ids,
"note_paths": [item.note_path for item in analyses],
"pdf_paths": [item.pdf_path for item in analyses],
"digest_path": digest_rel,
"source_counts": self._state("source_counts"),
"excluded_yesterday_count": len(

2
tests/unit/test_package_versions.py
View file

@ -224,7 +224,7 @@ def test_daily_paper_declares_runtime_dependencies() -> None:
assert not by_name["reme-ai"].extras
assert Version("0.4.1.11") not in by_name["reme-ai"].specifier
assert Version("0.4.1.12") in by_name["reme-ai"].specifier
assert Version("0.4.2.0") in by_name["reme-ai"].specifier
assert "pypdf" in by_name