fix(daily-paper): sanitize Unicode surrogates (#487)

* fix(daily-paper): sanitize Unicode surrogates

* fix(daily-paper): sanitize analysis workflow state

reme/steps/cookbook/daily_paper/_common.py

@@ -22,6 +22,7 @@ _FRONTMATTER_PATTERN = re.compile(r"^---\s*\n.*?\n---\s*\n", re.DOTALL)
 _MARKDOWN_HEADING_PATTERN = re.compile(r"^#+\s*")
 _UNSAFE_FILENAME_CHARS = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
 _CHINESE_PATTERN = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]")
+_SURROGATE_PATTERN = re.compile(r"[\ud800-\udfff]")
 _OutputT = TypeVar("_OutputT", bound=BaseModel)
 
 
@@ -36,9 +37,28 @@ def strip_frontmatter(body: str) -> str:
     return _FRONTMATTER_PATTERN.sub("", body.strip(), count=1).strip()
 
 
+def replace_surrogates(value: str) -> str:
+    """Replace invalid Unicode surrogate code points with replacement characters."""
+    return _SURROGATE_PATTERN.sub("\ufffd", value)
+
+
+def _replace_surrogates_recursive(value: Any) -> Any:
+    """Replace surrogates in strings nested in frontmatter metadata."""
+    if isinstance(value, str):
+        return replace_surrogates(value)
+    if isinstance(value, dict):
+        return {_replace_surrogates_recursive(key): _replace_surrogates_recursive(item) for key, item in value.items()}
+    if isinstance(value, list):
+        return [_replace_surrogates_recursive(item) for item in value]
+    if isinstance(value, tuple):
+        return tuple(_replace_surrogates_recursive(item) for item in value)
+    return value
+
+
 def normalize_chinese_title(raw: str, fallback: str) -> str:
     """Return one safe Chinese title that can also be used as the filename stem."""
-    title = _MARKDOWN_HEADING_PATTERN.sub("", str(raw or "").strip())
+    title = replace_surrogates(str(raw or "").strip())
+    title = _MARKDOWN_HEADING_PATTERN.sub("", title)
     if title.lower().endswith(".md"):
         title = title[:-3]
     title = _UNSAFE_FILENAME_CHARS.sub("-", title)
@@ -92,7 +112,7 @@ async def write_atomic(path: Path, content: str | bytes) -> None:
     lock = await get_path_lock(path)
     async with lock:
         temp_path = path.with_name(f".{path.name}.{uuid4().hex}.tmp")
-        payload = content.encode("utf-8") if isinstance(content, str) else content
+        payload = replace_surrogates(content).encode("utf-8") if isinstance(content, str) else content
         try:
             async with aiofiles.open(temp_path, "wb") as stream:
                 await stream.write(payload)
@@ -104,7 +124,9 @@ async def write_atomic(path: Path, content: str | bytes) -> None:
 
 async def write_markdown(path: Path, body: str, metadata: dict[str, Any]) -> None:
     """Serialize a frontmatter Markdown document atomically."""
-    rendered = frontmatter.dumps(frontmatter.Post(body.strip(), **metadata))
+    safe_body = replace_surrogates(body).strip()
+    safe_metadata = _replace_surrogates_recursive(metadata)
+    rendered = frontmatter.dumps(frontmatter.Post(safe_body, **safe_metadata))
     await write_atomic(path, rendered if rendered.endswith("\n") else f"{rendered}\n")
 
 

reme/steps/cookbook/daily_paper/analyze.py

@@ -12,6 +12,7 @@ from ._common import (
     DailyPaperStep,
     iter_note_metadata,
     normalize_chinese_title,
+    replace_surrogates,
     resolve_unique_note_path,
     strip_frontmatter,
     structured_output,
@@ -43,7 +44,8 @@ class DailyPaperAnalyzeStep(DailyPaperStep):
         page_count = min(len(reader.pages), max_pages)
         truncated = len(reader.pages) > max_pages
         for page_number, page in enumerate(reader.pages[:page_count], start=1):
-            block = f"\n\n--- PAGE {page_number} ---\n\n{(page.extract_text() or '').strip()}"
+            page_text = replace_surrogates((page.extract_text() or "").strip())
+            block = f"\n\n--- PAGE {page_number} ---\n\n{page_text}"
             if size + len(block) > max_chars:
                 if (remaining := max_chars - size) > 0:
                     chunks.append(block[:remaining])
@@ -126,8 +128,9 @@ class DailyPaperAnalyzeStep(DailyPaperStep):
         )
         used_titles.add(title)
         note_rel = note_path.relative_to(self.workspace_path).as_posix()
-        body = strip_frontmatter(output.body)
-        if not output.desc.strip() or not body:
+        desc = replace_surrogates(output.desc.strip())
+        body = replace_surrogates(strip_frontmatter(output.body))
+        if not desc or not body:
             raise ValueError(f"Agent returned an empty paper note for {paper.arxiv_id}")
         await write_markdown(
             note_path,
@@ -135,7 +138,7 @@ class DailyPaperAnalyzeStep(DailyPaperStep):
             {
                 "name": title,
                 "title": title,
-                "description": output.desc.strip(),
+                "description": desc,
                 "kind": "daily-paper-analysis",
                 "arxiv_id": paper.arxiv_id,
                 "source_title": paper.title,
@@ -163,7 +166,7 @@ class DailyPaperAnalyzeStep(DailyPaperStep):
             arxiv_id=paper.arxiv_id,
             reasoning=selected.reasoning,
             title=title,
-            desc=output.desc.strip(),
+            desc=desc,
             body=body,
             note_path=note_rel,
             pdf_path=pdf_rel,

tests/unit/test_daily_paper.py

@@ -25,6 +25,12 @@ from reme.steps.cookbook.daily_paper import (
     DailyPaperSelectStep,
 )
 from reme.steps.cookbook.daily_paper import analyze, collect
+from reme.steps.cookbook.daily_paper._common import (
+    normalize_chinese_title,
+    replace_surrogates,
+    write_atomic,
+    write_markdown,
+)
 from reme.steps.cookbook.daily_paper.rank import build_candidate_pool, rrf_score
 from reme.steps.cookbook.dingtalk import DingTalkMarkdownSendStep
 from reme.steps.cookbook.dingtalk import send as dingtalk_send
@@ -57,6 +63,73 @@ def _paper(arxiv_id: str, *, title: str = "Paper", upvotes: int = 10) -> PaperIn
     )
 
 
+def test_daily_paper_replaces_surrogates_in_text_and_titles():
+    """Invalid surrogate code points become visible replacement characters."""
+    assert replace_surrogates("before\ud800middle\udfffafter") == "before\ufffdmiddle\ufffdafter"
+    assert normalize_chinese_title("论文\ud800标题", "fallback") == "论文\ufffd标题"
+
+
+@pytest.mark.asyncio
+async def test_daily_paper_atomic_write_replaces_surrogates(tmp_path: Path):
+    """Markdown writes always produce valid UTF-8 even when model output is malformed."""
+    target = tmp_path / "note.md"
+
+    await write_atomic(target, "before\ud800after")
+
+    assert target.read_text(encoding="utf-8") == "before\ufffdafter"
+
+
+@pytest.mark.asyncio
+async def test_daily_paper_markdown_write_replaces_surrogates_in_frontmatter(tmp_path: Path):
+    """Frontmatter serialization sanitizes nested metadata before YAML encoding."""
+    target = tmp_path / "note.md"
+
+    await write_markdown(
+        target,
+        "body\ud800text",
+        {"description": "meta\udffftext", "authors": ["safe", "author\ud800name"]},
+    )
+
+    post = frontmatter.load(target)
+    assert post.content == "body\ufffdtext"
+    assert post.metadata == {
+        "description": "meta\ufffdtext",
+        "authors": ["safe", "author\ufffdname"],
+    }
+
+
+def test_daily_paper_pdf_extraction_replaces_surrogates(monkeypatch, tmp_path: Path):
+    """Malformed PDF text is sanitized before it enters an agent prompt."""
+
+    class FakePage:
+        """Return text containing one unpaired surrogate."""
+
+        @staticmethod
+        def extract_text():
+            """Return the malformed page text."""
+            return "before\ud800after"
+
+    class FakeReader:
+        """Expose one fake PDF page."""
+
+        def __init__(self, _path: str):
+            self.pages = [FakePage()]
+
+    import pypdf
+
+    monkeypatch.setattr(pypdf, "PdfReader", FakeReader)
+
+    content, page_count, truncated = DailyPaperAnalyzeStep._extract_pdf_text_sync(  # pylint: disable=protected-access
+        tmp_path / "paper.pdf",
+        20,
+        300_000,
+    )
+
+    assert content == "--- PAGE 1 ---\n\nbefore\ufffdafter"
+    assert page_count == 1
+    assert truncated is False
+
+
 def test_hf_payload_and_html_normalization():
     """HF list/detail shapes normalize and HTML rank order de-duplicates."""
     payload = {
@@ -705,8 +778,8 @@ async def test_pipeline_filters_strict_yesterday_and_writes_outputs(
             },
             {
                 "title": "记忆代理研究",
-                "desc": "Detailed note one",
-                "body": "Evidence one [p. 1].",
+                "desc": "Detailed note\ud800 one",
+                "body": "Evidence\udfff one [p. 1].",
             },
             {
                 "title": "上下文压缩研究",
@@ -801,7 +874,9 @@ async def test_pipeline_filters_strict_yesterday_and_writes_outputs(
     assert "ReMe" not in analysis_prompt
     assert "# PDF 分页文本" in analysis_prompt
     digest_prompt = cc_wrapper.calls[-1]["inputs"]
-    assert "Evidence one [p. 1]." in digest_prompt
+    assert "Evidence\ufffd one [p. 1]." in digest_prompt
+    assert "Detailed note\ufffd one" in digest_prompt
+    assert not any("\ud800" <= character <= "\udfff" for character in digest_prompt)
     assert "调用 Read" not in digest_prompt
     assert "daily/2026-07-21" not in digest_prompt
     assert "长期记忆" not in digest_prompt