From a721218df01dc7ad5d021d68ec9faed785179425 Mon Sep 17 00:00:00 2001
From: adarshh347 <adarsh347k@gmail.com>
Date: Thu, 5 Nov 2026 13:52:11 +0530
Subject: [PATCH] Retry empty Daily Paper frontmatter output

---
 .../daily_paper/src/reme_daily_paper/base.py  |  3 +-
 .../src/reme_daily_paper/digest.py            |  2 +-
 .../tests/test_structured_recovery.py         | 44 +++++++++++++++++++
 tests/unit/test_package_versions.py           |  1 +
 4 files changed, 47 insertions(+), 3 deletions(-)

diff --git a/plugins/daily_paper/src/reme_daily_paper/base.py b/plugins/daily_paper/src/reme_daily_paper/base.py
index 3301d153..07b25556 100644
--- a/plugins/daily_paper/src/reme_daily_paper/base.py
+++ b/plugins/daily_paper/src/reme_daily_paper/base.py
@@ -19,7 +19,7 @@ from .schema import AnalyzedPaper, DailyPaperMarkdownOutput
 # Number of papers selected, analyzed, and digested each run. Shared across steps.
 PAPER_COUNT = 3
 _STATE_PREFIX = "daily_paper_"
-_FRONTMATTER_PATTERN = re.compile(r"^---\s*\n.*?\n---(?:\s*\n|$)", re.DOTALL)
+_FRONTMATTER_PATTERN = re.compile(r"^---[ \t]*\n.*?^---[ \t]*(?:\n|$)", re.DOTALL | re.MULTILINE)
 _MARKDOWN_HEADING_PATTERN = re.compile(r"^#+\s*")
 _UNSAFE_FILENAME_CHARS = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
 _CHINESE_PATTERN = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]")
@@ -169,7 +169,6 @@ async def reserve_unique_note_path(
 
 async def write_atomic(path: Path, content: str | bytes) -> None:
     """Write atomically under the path lock, cleaning surrogate text first."""
-    path.parent.mkdir(parents=True, exist_ok=True)
     lock = await get_path_lock(path)
     async with lock:
         payload = replace_surrogates(content) if isinstance(content, str) else content
diff --git a/plugins/daily_paper/src/reme_daily_paper/digest.py b/plugins/daily_paper/src/reme_daily_paper/digest.py
index 51973ab1..fb0563f0 100644
--- a/plugins/daily_paper/src/reme_daily_paper/digest.py
+++ b/plugins/daily_paper/src/reme_daily_paper/digest.py
@@ -164,7 +164,7 @@ class DailyPaperDigestStep(DailyPaperStep):
                 {
                     "name": title,
                     "title": title,
-                    "description": output.desc.strip(),
+                    "description": output.desc,
                     "kind": "daily-paper-brief",
                     "date": day,
                     "arxiv_ids": selected_ids,
diff --git a/plugins/daily_paper/tests/test_structured_recovery.py b/plugins/daily_paper/tests/test_structured_recovery.py
index bbbdc1d6..5ff4cda3 100644
--- a/plugins/daily_paper/tests/test_structured_recovery.py
+++ b/plugins/daily_paper/tests/test_structured_recovery.py
@@ -10,6 +10,7 @@ import pytest
 from pydantic import BaseModel
 
 from reme_daily_paper.analyze import DailyPaperAnalyzeStep
+from reme_daily_paper.base import markdown_output
 from reme_daily_paper.schema import PaperInfo, PaperPick
 from reme.components import ApplicationContext, RuntimeContext
 from reme.components.agent_wrapper import AsAgentWrapper
@@ -134,3 +135,46 @@ async def test_codex_json_parse_failure_gets_one_shorter_retry(tmp_path, monkeyp
     assert instruction not in thread.run.await_args_list[0].args[0]
     assert instruction in thread.run.await_args_list[1].args[0]
     downloader.download.assert_awaited_once()
+
+
+@pytest.mark.parametrize("empty_frontmatter", ["---\n---", "---\n---\n"])
+@pytest.mark.asyncio
+async def test_empty_frontmatter_retries_once_without_repeating_pdf_work(tmp_path, monkeypatch, empty_frontmatter):
+    """An empty frontmatter-only note triggers one shorter retry after one PDF extraction."""
+    app = ApplicationContext(workspace_dir=str(tmp_path))
+    valid = {"title": "论文摘要", "desc": "summary", "body": "Recovered result"}
+    reply = AsyncMock(
+        side_effect=[
+            {"structured_output": {"title": "论文摘要", "desc": "summary", "body": empty_frontmatter}},
+            {"structured_output": valid},
+        ],
+    )
+    step = DailyPaperAnalyzeStep(app_context=app)
+    step.agent_wrapper = SimpleNamespace(reply=reply)
+    step.context = RuntimeContext(daily_paper_run_date="2026-07-21")
+    extraction_calls = []
+
+    def extract(*_args):
+        extraction_calls.append(True)
+        return "Evidence [p. 1]", 1, False
+
+    monkeypatch.setattr(step, "_extract_pdf_text_sync", extract)
+    downloader = SimpleNamespace(download=AsyncMock())
+    paper = PaperInfo(arxiv_id="2607.10001")
+    pick = PaperPick(arxiv_id=paper.arxiv_id, reasoning="test")
+
+    result = await step._analyze_one(downloader, paper, pick, set())
+
+    note = (tmp_path / result.note_path).read_text(encoding="utf-8")
+    assert "Recovered result" in note
+    assert reply.await_count == 2
+    downloader.download.assert_awaited_once()
+    assert extraction_calls == [True]
+
+
+@pytest.mark.parametrize("body", ["---\n---\nUseful note", "---\n---\n\nUseful note"])
+def test_content_after_empty_frontmatter_is_retained(body):
+    """Removing an empty frontmatter block must leave the Markdown body intact."""
+    result = markdown_output({"structured_output": {"title": "Title", "desc": "Description", "body": body}})
+
+    assert result.body == "Useful note"
diff --git a/tests/unit/test_package_versions.py b/tests/unit/test_package_versions.py
index 70ff73a8..d8d89277 100644
--- a/tests/unit/test_package_versions.py
+++ b/tests/unit/test_package_versions.py
@@ -224,6 +224,7 @@ def test_daily_paper_declares_runtime_dependencies() -> None:
 
     assert not by_name["reme-ai"].extras
     assert Version("0.4.1.11") not in by_name["reme-ai"].specifier
+    assert Version("0.4.1.12") not in by_name["reme-ai"].specifier
     assert Version("0.4.2.0") in by_name["reme-ai"].specifier
     assert "pypdf" in by_name