fix(daily-paper): avoid overwriting colliding notes (#699)
Some checks failed
CI / Python packages / Build and verify distributions (push) Waiting to run
CI / Python quality / GitHub Actions (push) Waiting to run
CI / Python quality / Pre-commit (push) Waiting to run
CI / Python tests / Unit Tests - py3.11 (push) Waiting to run
CI / Python tests / Unit Tests - py3.12 (push) Waiting to run
CI / Python tests / Unit Tests - py3.13 (push) Waiting to run
CI / Python tests / Unit Tests - py3.14 (push) Waiting to run
CI / Windows / CLI smoke - py3.11 (push) Waiting to run
CI and Release / Docker / Build and test / amd64 (push) Waiting to run
CI and Release / Docker / Build and test / arm64 (push) Waiting to run
CI and Release / Docker / Publish multi-platform tags (push) Blocked by required conditions
Security / CodeQL / Analyze javascript-typescript (push) Waiting to run
Security / CodeQL / Analyze python (push) Waiting to run
Deploy / Documentation / Build documentation (push) Has been cancelled
Deploy / Documentation / deploy (push) Has been cancelled

* fix(daily-paper): avoid overwriting colliding notes

* fix(daily-paper): serialize unique note publication
This commit is contained in:
Adarsh 2026-11-02 10:41:59 +05:30 • committed by GitHub
parent 7f2fa864c1
commit db84ebd1a6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 205 additions and 64 deletions

View file

@ -11,7 +11,7 @@ from .base import (
iter_note_metadata,
normalize_chinese_title,
replace_surrogates,
resolve_unique_note_path,
reserve_unique_note_path,
strip_frontmatter,
structured_output,
utc_now_iso,
@ -116,45 +116,45 @@ class DailyPaperAnalyzeStep(DailyPaperStep):
day_dir = self.workspace_path / daily_dir / day
existing_note = self._find_existing_note(day_dir, paper.arxiv_id)
suffix = f"({paper.arxiv_id})"
title, note_path = resolve_unique_note_path(
async with reserve_unique_note_path(
day_dir,
title,
taken=used_titles,
taken_suffix=suffix,
disk_suffix=suffix,
existing=existing_note,
)
used_titles.add(title)
note_rel = note_path.relative_to(self.workspace_path).as_posix()
desc = replace_surrogates(output.desc.strip())
body = replace_surrogates(strip_frontmatter(output.body))
if not desc or not body:
raise ValueError(f"Agent returned an empty paper note for {paper.arxiv_id}")
await write_markdown(
note_path,
body,
{
"name": title,
"title": title,
"description": desc,
"kind": "daily-paper-analysis",
"arxiv_id": paper.arxiv_id,
"source_title": paper.title,
"authors": paper.authors,
"hf_url": paper.hf_url,
"arxiv_url": paper.arxiv_url,
"download_url": paper.pdf_url,
"source_pdf": f"[[{pdf_rel}]]",
"published_at": paper.published_at,
"monthly_rank": paper.monthly_rank,
"weekly_rank": paper.weekly_rank,
"fused_score": round(paper.fused_score, 8),
"selection_reasoning": selected.reasoning,
"generated_at": utc_now_iso(),
"pdf_pages": page_count,
"pdf_text_truncated": truncated,
},
)
) as (title, note_path):
used_titles.add(title)
note_rel = note_path.relative_to(self.workspace_path).as_posix()
desc = replace_surrogates(output.desc.strip())
body = replace_surrogates(strip_frontmatter(output.body))
if not desc or not body:
raise ValueError(f"Agent returned an empty paper note for {paper.arxiv_id}")
await write_markdown(
note_path,
body,
{
"name": title,
"title": title,
"description": desc,
"kind": "daily-paper-analysis",
"arxiv_id": paper.arxiv_id,
"source_title": paper.title,
"authors": paper.authors,
"hf_url": paper.hf_url,
"arxiv_url": paper.arxiv_url,
"download_url": paper.pdf_url,
"source_pdf": f"[[{pdf_rel}]]",
"published_at": paper.published_at,
"monthly_rank": paper.monthly_rank,
"weekly_rank": paper.weekly_rank,
"fused_score": round(paper.fused_score, 8),
"selection_reasoning": selected.reasoning,
"generated_at": utc_now_iso(),
"pdf_pages": page_count,
"pdf_text_truncated": truncated,
},
)
changes = list(self.context.get("changes") or [])
if existing_note is not None and existing_note != note_path:
existing_rel = existing_note.relative_to(self.workspace_path).as_posix()
@ -181,6 +181,7 @@ class DailyPaperAnalyzeStep(DailyPaperStep):
)
async def execute(self):
"""Analyze selected papers and persist their notes."""
assert self.context is not None
self.context["changes"] = []
if self._skip():

View file

@ -5,6 +5,7 @@ import os
import re
import zoneinfo
from collections.abc import Iterator
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any, TypeVar
from uuid import uuid4
@ -107,14 +108,39 @@ def resolve_unique_note_path(
disk_suffix: str,
existing: Path | None,
) -> tuple[str, Path]:
"""Disambiguate a note title against already-used titles and on-disk collisions."""
if title in taken:
title = f"{title}{taken_suffix}"
path = day_dir / f"{title}.md"
if path.exists() and path != existing:
title = f"{title}{disk_suffix}"
path = day_dir / f"{title}.md"
return title, path
"""Choose a note path without replacing another note after repeated collisions."""
candidate = title
if candidate in taken:
candidate = f"{candidate}{taken_suffix}"
while True:
path = day_dir / f"{candidate}.md"
if candidate not in taken and (path == existing or not path.exists()):
return candidate, path
candidate = f"{candidate}{disk_suffix}"
@asynccontextmanager
async def reserve_unique_note_path(
day_dir: Path,
title: str,
*,
taken: set[str],
taken_suffix: str,
disk_suffix: str,
existing: Path | None,
):
"""Serialize path selection and publication for one Daily Paper directory."""
lock = await get_path_lock(day_dir)
async with lock:
yield resolve_unique_note_path(
day_dir,
title,
taken=taken,
taken_suffix=taken_suffix,
disk_suffix=disk_suffix,
existing=existing,
)
async def write_atomic(path: Path, content: str | bytes) -> None:

View file

@ -12,7 +12,7 @@ from .base import (
PAPER_COUNT,
DailyPaperStep,
normalize_chinese_title,
resolve_unique_note_path,
reserve_unique_note_path,
strip_frontmatter,
structured_output,
utc_now_iso,
@ -109,33 +109,33 @@ class DailyPaperDigestStep(DailyPaperStep):
existing_rel = str(self._state("existing_digest_path") or "").strip()
existing_path = self.workspace_path / existing_rel if existing_rel else None
existing_was_file = existing_path is not None and existing_path.is_file()
title, digest_path = resolve_unique_note_path(
async with reserve_unique_note_path(
self.workspace_path / daily_dir / day,
title,
taken={item.title for item in analyses},
taken_suffix="(每日简报)",
disk_suffix=f"({day})",
existing=existing_path,
)
digest_rel = digest_path.relative_to(self.workspace_path).as_posix()
body = self._validate_historical_wikilinks(body, run_day, digest_path)
body += "\n\n## 详细论文\n\n" + "\n".join(f"- {link}" for link in wikilinks)
selected_ids = [item.arxiv_id for item in analyses]
await write_markdown(
digest_path,
body,
{
"name": title,
"title": title,
"description": output.desc.strip(),
"kind": "daily-paper-brief",
"date": day,
"arxiv_ids": selected_ids,
"selection_reasoning": [item.reasoning for item in analyses],
"source_notes": wikilinks,
"generated_at": utc_now_iso(),
},
)
) as (title, digest_path):
digest_rel = digest_path.relative_to(self.workspace_path).as_posix()
body = self._validate_historical_wikilinks(body, run_day, digest_path)
body += "\n\n## 详细论文\n\n" + "\n".join(f"- {link}" for link in wikilinks)
selected_ids = [item.arxiv_id for item in analyses]
await write_markdown(
digest_path,
body,
{
"name": title,
"title": title,
"description": output.desc.strip(),
"kind": "daily-paper-brief",
"date": day,
"arxiv_ids": selected_ids,
"selection_reasoning": [item.reasoning for item in analyses],
"source_notes": wikilinks,
"generated_at": utc_now_iso(),
},
)
changes = list(self.context.get("changes") or [])
if existing_was_file and existing_path != digest_path:
existing_path.unlink()

View file

@ -0,0 +1,114 @@
"""Filename selection must preserve notes when generated titles collide."""
import asyncio
import frontmatter
import pytest
from reme_daily_paper.base import reserve_unique_note_path, resolve_unique_note_path, write_markdown
@pytest.mark.asyncio
@pytest.mark.parametrize("suffix", ["(2610.12345)", "(2026-10-31)"])
async def test_repeated_disk_collision_preserves_existing_notes(tmp_path, suffix):
"""A second occupied candidate must not be replaced."""
title = "论文总结"
occupied = [tmp_path / f"{title}.md", tmp_path / f"{title}{suffix}.md"]
for path in occupied:
path.write_text("unrelated user-owned note", encoding="utf-8")
selected_title, destination = resolve_unique_note_path(
tmp_path,
title,
taken=set(),
taken_suffix=suffix,
disk_suffix=suffix,
existing=None,
)
await write_markdown(destination, "new generated content", {"kind": "daily-paper-analysis"})
assert selected_title == f"{title}{suffix}{suffix}"
assert destination == tmp_path / f"{selected_title}.md"
assert all(path.read_text(encoding="utf-8") == "unrelated user-owned note" for path in occupied)
def test_reserved_title_and_disk_collision_both_advance_candidate(tmp_path):
"""Skip names reserved by this run as well as files on disk."""
title = "论文总结"
suffix = "(paper-id)"
first = f"{title}{suffix}"
second = f"{first}{suffix}"
(tmp_path / f"{first}.md").write_text("existing note", encoding="utf-8")
selected_title, destination = resolve_unique_note_path(
tmp_path,
title,
taken={title, first, second},
taken_suffix=suffix,
disk_suffix=suffix,
existing=None,
)
assert selected_title == f"{second}{suffix}"
assert destination == tmp_path / f"{selected_title}.md"
assert not destination.exists()
def test_explicitly_owned_note_remains_updateable(tmp_path):
"""The matched note for the same paper remains an update target."""
path = tmp_path / "论文总结.md"
path.write_text("old owned note", encoding="utf-8")
selected_title, destination = resolve_unique_note_path(
tmp_path,
"论文总结",
taken=set(),
taken_suffix="(id)",
disk_suffix="(id)",
existing=path,
)
assert selected_title == "论文总结"
assert destination == path
def test_single_collision_keeps_existing_suffix_convention(tmp_path):
"""The first collision still uses the established suffix."""
title = "论文总结"
suffix = "(paper-id)"
(tmp_path / f"{title}.md").write_text("unrelated note", encoding="utf-8")
selected_title, destination = resolve_unique_note_path(
tmp_path,
title,
taken=set(),
taken_suffix=suffix,
disk_suffix=suffix,
existing=None,
)
assert selected_title == f"{title}{suffix}"
assert destination == tmp_path / f"{selected_title}.md"
assert not destination.exists()
@pytest.mark.asyncio
async def test_concurrent_notes_reserve_distinct_paths(tmp_path):
"""Concurrent runs select and publish under one directory-scoped lock."""
async def create_note(arxiv_id):
async with reserve_unique_note_path(
tmp_path,
"论文总结",
taken=set(),
taken_suffix=f"({arxiv_id})",
disk_suffix=f"({arxiv_id})",
existing=None,
) as (title, path):
await write_markdown(path, f"body from {arxiv_id}", {"arxiv_id": arxiv_id, "title": title})
return path
paths = await asyncio.gather(create_note("paper-a"), create_note("paper-b"))
assert len(set(paths)) == 2
assert {frontmatter.load(path).metadata["arxiv_id"] for path in paths} == {"paper-a", "paper-b"}