diff --git a/reme/steps/cookbook/daily_paper/_common.py b/reme/steps/cookbook/daily_paper/_common.py index 7d8dcd33..72ec0a90 100644 --- a/reme/steps/cookbook/daily_paper/_common.py +++ b/reme/steps/cookbook/daily_paper/_common.py @@ -22,6 +22,7 @@ _FRONTMATTER_PATTERN = re.compile(r"^---\s*\n.*?\n---\s*\n", re.DOTALL) _MARKDOWN_HEADING_PATTERN = re.compile(r"^#+\s*") _UNSAFE_FILENAME_CHARS = re.compile(r'[<>:"/\\|?*\x00-\x1f]') _CHINESE_PATTERN = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]") +_SURROGATE_PATTERN = re.compile(r"[\ud800-\udfff]") _OutputT = TypeVar("_OutputT", bound=BaseModel) @@ -36,9 +37,28 @@ def strip_frontmatter(body: str) -> str: return _FRONTMATTER_PATTERN.sub("", body.strip(), count=1).strip() +def replace_surrogates(value: str) -> str: + """Replace invalid Unicode surrogate code points with replacement characters.""" + return _SURROGATE_PATTERN.sub("\ufffd", value) + + +def _replace_surrogates_recursive(value: Any) -> Any: + """Replace surrogates in strings nested in frontmatter metadata.""" + if isinstance(value, str): + return replace_surrogates(value) + if isinstance(value, dict): + return {_replace_surrogates_recursive(key): _replace_surrogates_recursive(item) for key, item in value.items()} + if isinstance(value, list): + return [_replace_surrogates_recursive(item) for item in value] + if isinstance(value, tuple): + return tuple(_replace_surrogates_recursive(item) for item in value) + return value + + def normalize_chinese_title(raw: str, fallback: str) -> str: """Return one safe Chinese title that can also be used as the filename stem.""" - title = _MARKDOWN_HEADING_PATTERN.sub("", str(raw or "").strip()) + title = replace_surrogates(str(raw or "").strip()) + title = _MARKDOWN_HEADING_PATTERN.sub("", title) if title.lower().endswith(".md"): title = title[:-3] title = _UNSAFE_FILENAME_CHARS.sub("-", title) @@ -92,7 +112,7 @@ async def write_atomic(path: Path, content: str | bytes) -> None: lock = await get_path_lock(path) async with lock: temp_path = path.with_name(f".{path.name}.{uuid4().hex}.tmp") - payload = content.encode("utf-8") if isinstance(content, str) else content + payload = replace_surrogates(content).encode("utf-8") if isinstance(content, str) else content try: async with aiofiles.open(temp_path, "wb") as stream: await stream.write(payload) @@ -104,7 +124,9 @@ async def write_atomic(path: Path, content: str | bytes) -> None: async def write_markdown(path: Path, body: str, metadata: dict[str, Any]) -> None: """Serialize a frontmatter Markdown document atomically.""" - rendered = frontmatter.dumps(frontmatter.Post(body.strip(), **metadata)) + safe_body = replace_surrogates(body).strip() + safe_metadata = _replace_surrogates_recursive(metadata) + rendered = frontmatter.dumps(frontmatter.Post(safe_body, **safe_metadata)) await write_atomic(path, rendered if rendered.endswith("\n") else f"{rendered}\n") diff --git a/reme/steps/cookbook/daily_paper/analyze.py b/reme/steps/cookbook/daily_paper/analyze.py index 6d1f177b..68e3579f 100644 --- a/reme/steps/cookbook/daily_paper/analyze.py +++ b/reme/steps/cookbook/daily_paper/analyze.py @@ -12,6 +12,7 @@ from ._common import ( DailyPaperStep, iter_note_metadata, normalize_chinese_title, + replace_surrogates, resolve_unique_note_path, strip_frontmatter, structured_output, @@ -43,7 +44,8 @@ class DailyPaperAnalyzeStep(DailyPaperStep): page_count = min(len(reader.pages), max_pages) truncated = len(reader.pages) > max_pages for page_number, page in enumerate(reader.pages[:page_count], start=1): - block = f"\n\n--- PAGE {page_number} ---\n\n{(page.extract_text() or '').strip()}" + page_text = replace_surrogates((page.extract_text() or "").strip()) + block = f"\n\n--- PAGE {page_number} ---\n\n{page_text}" if size + len(block) > max_chars: if (remaining := max_chars - size) > 0: chunks.append(block[:remaining]) @@ -126,8 +128,9 @@ class DailyPaperAnalyzeStep(DailyPaperStep): ) used_titles.add(title) note_rel = note_path.relative_to(self.workspace_path).as_posix() - body = strip_frontmatter(output.body) - if not output.desc.strip() or not body: + desc = replace_surrogates(output.desc.strip()) + body = replace_surrogates(strip_frontmatter(output.body)) + if not desc or not body: raise ValueError(f"Agent returned an empty paper note for {paper.arxiv_id}") await write_markdown( note_path, @@ -135,7 +138,7 @@ class DailyPaperAnalyzeStep(DailyPaperStep): { "name": title, "title": title, - "description": output.desc.strip(), + "description": desc, "kind": "daily-paper-analysis", "arxiv_id": paper.arxiv_id, "source_title": paper.title, @@ -163,7 +166,7 @@ class DailyPaperAnalyzeStep(DailyPaperStep): arxiv_id=paper.arxiv_id, reasoning=selected.reasoning, title=title, - desc=output.desc.strip(), + desc=desc, body=body, note_path=note_rel, pdf_path=pdf_rel, diff --git a/tests/unit/test_daily_paper.py b/tests/unit/test_daily_paper.py index e0a027a3..851a93da 100644 --- a/tests/unit/test_daily_paper.py +++ b/tests/unit/test_daily_paper.py @@ -25,6 +25,12 @@ from reme.steps.cookbook.daily_paper import ( DailyPaperSelectStep, ) from reme.steps.cookbook.daily_paper import analyze, collect +from reme.steps.cookbook.daily_paper._common import ( + normalize_chinese_title, + replace_surrogates, + write_atomic, + write_markdown, +) from reme.steps.cookbook.daily_paper.rank import build_candidate_pool, rrf_score from reme.steps.cookbook.dingtalk import DingTalkMarkdownSendStep from reme.steps.cookbook.dingtalk import send as dingtalk_send @@ -57,6 +63,73 @@ def _paper(arxiv_id: str, *, title: str = "Paper", upvotes: int = 10) -> PaperIn ) +def test_daily_paper_replaces_surrogates_in_text_and_titles(): + """Invalid surrogate code points become visible replacement characters.""" + assert replace_surrogates("before\ud800middle\udfffafter") == "before\ufffdmiddle\ufffdafter" + assert normalize_chinese_title("论文\ud800标题", "fallback") == "论文\ufffd标题" + + +@pytest.mark.asyncio +async def test_daily_paper_atomic_write_replaces_surrogates(tmp_path: Path): + """Markdown writes always produce valid UTF-8 even when model output is malformed.""" + target = tmp_path / "note.md" + + await write_atomic(target, "before\ud800after") + + assert target.read_text(encoding="utf-8") == "before\ufffdafter" + + +@pytest.mark.asyncio +async def test_daily_paper_markdown_write_replaces_surrogates_in_frontmatter(tmp_path: Path): + """Frontmatter serialization sanitizes nested metadata before YAML encoding.""" + target = tmp_path / "note.md" + + await write_markdown( + target, + "body\ud800text", + {"description": "meta\udffftext", "authors": ["safe", "author\ud800name"]}, + ) + + post = frontmatter.load(target) + assert post.content == "body\ufffdtext" + assert post.metadata == { + "description": "meta\ufffdtext", + "authors": ["safe", "author\ufffdname"], + } + + +def test_daily_paper_pdf_extraction_replaces_surrogates(monkeypatch, tmp_path: Path): + """Malformed PDF text is sanitized before it enters an agent prompt.""" + + class FakePage: + """Return text containing one unpaired surrogate.""" + + @staticmethod + def extract_text(): + """Return the malformed page text.""" + return "before\ud800after" + + class FakeReader: + """Expose one fake PDF page.""" + + def __init__(self, _path: str): + self.pages = [FakePage()] + + import pypdf + + monkeypatch.setattr(pypdf, "PdfReader", FakeReader) + + content, page_count, truncated = DailyPaperAnalyzeStep._extract_pdf_text_sync( # pylint: disable=protected-access + tmp_path / "paper.pdf", + 20, + 300_000, + ) + + assert content == "--- PAGE 1 ---\n\nbefore\ufffdafter" + assert page_count == 1 + assert truncated is False + + def test_hf_payload_and_html_normalization(): """HF list/detail shapes normalize and HTML rank order de-duplicates.""" payload = { @@ -705,8 +778,8 @@ async def test_pipeline_filters_strict_yesterday_and_writes_outputs( }, { "title": "记忆代理研究", - "desc": "Detailed note one", - "body": "Evidence one [p. 1].", + "desc": "Detailed note\ud800 one", + "body": "Evidence\udfff one [p. 1].", }, { "title": "上下文压缩研究", @@ -801,7 +874,9 @@ async def test_pipeline_filters_strict_yesterday_and_writes_outputs( assert "ReMe" not in analysis_prompt assert "# PDF 分页文本" in analysis_prompt digest_prompt = cc_wrapper.calls[-1]["inputs"] - assert "Evidence one [p. 1]." in digest_prompt + assert "Evidence\ufffd one [p. 1]." in digest_prompt + assert "Detailed note\ufffd one" in digest_prompt + assert not any("\ud800" <= character <= "\udfff" for character in digest_prompt) assert "调用 Read" not in digest_prompt assert "daily/2026-07-21" not in digest_prompt assert "长期记忆" not in digest_prompt