mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-10 08:28:22 +00:00
feat(daily): add date-range queries and pagination (#685)
This commit is contained in:
parent
363e497ae9
commit
c23cc64715
6 changed files with 453 additions and 28 deletions
|
|
@ -179,11 +179,19 @@ Daily notes:
|
|||
```bash
|
||||
reme write path=daily/2026-06-20/demo-session.md name=demo-session description="Demo session" content="Recorded content"
|
||||
reme daily_list
|
||||
reme daily_list start_date=2026-06-01 end_date=2026-06-20 page=1 page_size=20
|
||||
reme daily_reindex
|
||||
```
|
||||
|
||||
`write` can create a daily note directly. Run `daily_reindex` when the day's index needs to be refreshed.
|
||||
|
||||
`daily_list` is read-only. Without new parameters, it still lists all notes for `date` (default: today) and returns
|
||||
`{date, count, notes}`. Both range dates are required, inclusive, and cannot be combined with a non-empty `date`.
|
||||
Ranges sort by date descending, then path ascending. Ranges or an explicit `page`/`page_size` enable pagination:
|
||||
page defaults to 1, page size to 100 (maximum 1000), and metadata adds `page`, `page_size`, `total_notes`, `total_pages`.
|
||||
An empty range has zero pages; a page past the end returns no notes. Totals count readable, valid, authorized notes.
|
||||
Pagination limits output, not scanning; each request reflects the current files rather than a cross-page snapshot.
|
||||
|
||||
---
|
||||
|
||||
## Automatic Memory
|
||||
|
|
|
|||
|
|
@ -173,11 +173,18 @@ Daily note:
|
|||
```bash
|
||||
reme write path=daily/2026-06-20/demo-session.md name=demo-session description="Demo session" content="记录内容"
|
||||
reme daily_list
|
||||
reme daily_list start_date=2026-06-01 end_date=2026-06-20 page=1 page_size=20
|
||||
reme daily_reindex
|
||||
```
|
||||
|
||||
`write` 可直接创建 daily note;需要刷新当天索引时运行 `daily_reindex`。
|
||||
|
||||
`daily_list` 只读。未传新参数时,仍列出 `date`(默认今天)的全部笔记,返回 `{date, count, notes}`。
|
||||
日期范围必须同时提供起止日期,包含两端,且不能与非空的 `date` 混用;按日期降序、同日路径升序排列。
|
||||
日期范围或显式的 `page` / `page_size` 会启用分页:页码默认 1,每页默认 100 条、最多 1000 条,
|
||||
metadata 增加 `page`、`page_size`、`total_notes`、`total_pages`。空范围页数为 0,越界页返回空列表;
|
||||
总数只统计可读取、有效且有权限访问的笔记。分页限制返回量,不限制扫描量;每次查询读取当前文件,不提供跨页快照。
|
||||
|
||||
---
|
||||
|
||||
## 自动记忆
|
||||
|
|
|
|||
|
|
@ -518,14 +518,33 @@ jobs:
|
|||
|
||||
daily_list:
|
||||
backend: base
|
||||
description: "List notes under a single day."
|
||||
description: >-
|
||||
List daily notes without changing files or indexes. Defaults to all notes for today; date selects one day.
|
||||
Alternatively, start_date and end_date select an inclusive range, newest day first then path ascending.
|
||||
Range queries or explicit page/page_size opt into pagination (defaults: page 1, page_size 100).
|
||||
Metadata adds page, page_size, total_notes and total_pages; count and notes describe the returned page.
|
||||
parameters:
|
||||
type: object
|
||||
properties:
|
||||
date:
|
||||
type: string
|
||||
description: "YYYY-MM-DD; empty = today"
|
||||
description: "YYYY-MM-DD; empty = today when no range is given; cannot be combined with a range"
|
||||
default: ""
|
||||
start_date:
|
||||
type: string
|
||||
description: "inclusive YYYY-MM-DD range start; requires end_date; cannot be combined with date"
|
||||
end_date:
|
||||
type: string
|
||||
description: "inclusive YYYY-MM-DD range end; requires start_date"
|
||||
page:
|
||||
type: integer
|
||||
minimum: 1
|
||||
description: "optional 1-based page; defaults to 1 when paginating; past the end returns no notes"
|
||||
page_size:
|
||||
type: integer
|
||||
minimum: 1
|
||||
maximum: 1000
|
||||
description: "optional notes per page; defaults to 100 when paginating"
|
||||
steps:
|
||||
- backend: daily_list_step
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from pathlib import Path
|
|||
|
||||
import frontmatter
|
||||
|
||||
from ._path import resolve_path, validate_filename_component
|
||||
from ._path import _check_path_permission, resolve_path, validate_filename_component
|
||||
from ...utils import get_logger
|
||||
|
||||
logger = get_logger(log_to_file=False)
|
||||
|
|
@ -64,7 +64,7 @@ def _rebuild_body(body: str, notes_content: str) -> str:
|
|||
return f"{body.rstrip()}\n\n{block}\n" if body.strip() else f"{block}\n"
|
||||
|
||||
|
||||
def scan_notes(workspace_dir: Path, date: str, daily_dir: str) -> list[dict]:
|
||||
def scan_notes(workspace_dir: Path, date: str, daily_dir: str, *, allowed_paths=None) -> list[dict]:
|
||||
"""Walk ``<daily_dir>/<date>/*.md`` and pull each note's frontmatter.
|
||||
|
||||
Files under the date folder are named by note ``name``. Conversation
|
||||
|
|
@ -99,6 +99,8 @@ def scan_notes(workspace_dir: Path, date: str, daily_dir: str) -> list[dict]:
|
|||
target, err = resolve_path(workspace_dir, str(md_path))
|
||||
if err or target is None:
|
||||
continue
|
||||
if not _check_path_permission(workspace_dir, target, allowed_paths):
|
||||
continue
|
||||
if not target.is_file():
|
||||
continue
|
||||
except RuntimeError:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"""``daily_list`` — list the notes under a single day (pure read, no side effects).
|
||||
"""``daily_list`` — list daily notes with optional date ranges and pagination (pure read).
|
||||
|
||||
Returns one row per ``daily/<date>/<name>.md`` note file. The human answer
|
||||
contains the workspace-relative ``path`` plus every frontmatter key/value.
|
||||
|
|
@ -10,13 +10,15 @@ Response metadata includes ``notes`` as the original list of flat dicts:
|
|||
explicitly when the index page needs to be rebuilt. Decoupling
|
||||
read from write keeps each step's effect predictable.
|
||||
|
||||
Input is a single optional ``date`` (``YYYY-MM-DD``); falls back
|
||||
to today.
|
||||
Without range or pagination parameters, the optional ``date`` defaults to today
|
||||
and the original response shape is preserved. Ranges are inclusive and ordered
|
||||
newest day first, then by path. Pagination limits output, not scan I/O.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from ._daily_index import scan_notes
|
||||
from ._daily_index import parse_daily_date, scan_notes
|
||||
from ._path import resolve_path
|
||||
from ..base_step import BaseStep
|
||||
from ...components import R
|
||||
|
|
@ -25,16 +27,58 @@ from ...steps.evolve import now
|
|||
|
||||
@R.register("daily_list_step")
|
||||
class DailyListStep(BaseStep):
|
||||
"""List the notes under a single day. Pure read — no index refresh."""
|
||||
"""Read daily notes without refreshing indexes or changing workspace files."""
|
||||
|
||||
def _collect_params(self) -> tuple[str, str, Path]:
|
||||
"""Read ``date`` (default today, ``YYYY-MM-DD``), ``daily_dir``, and the workspace root."""
|
||||
def _collect_params(self) -> tuple[dict, int | None, int | None]:
|
||||
"""Keep legacy calls unpaged; range queries opt into bounded pages."""
|
||||
assert self.context is not None
|
||||
tz = self.app_context.app_config.timezone if self.app_context is not None else None
|
||||
day = self.context.get("date", "") or now(tz).strftime("%Y-%m-%d")
|
||||
daily_dir = self.config_value("daily_dir")
|
||||
workspace_dir = Path(self.file_store.workspace_path or ".").resolve()
|
||||
return day, daily_dir, workspace_dir
|
||||
ranged = any(key in self.context for key in ("start_date", "end_date"))
|
||||
if ranged:
|
||||
if self.context.get("date"):
|
||||
raise ValueError("date cannot be combined with start_date/end_date")
|
||||
scope = {key: self.context.get(key) for key in ("start_date", "end_date")}
|
||||
else:
|
||||
tz = self.app_context.app_config.timezone if self.app_context is not None else None
|
||||
scope = {"date": self.context.get("date") or now(tz).strftime("%Y-%m-%d")}
|
||||
for key, value in scope.items():
|
||||
if not isinstance(value, str) or parse_daily_date(value) != value:
|
||||
raise ValueError(f"{key} must be YYYY-MM-DD")
|
||||
if ranged and scope["start_date"] > scope["end_date"]:
|
||||
raise ValueError("start_date must not be later than end_date")
|
||||
if not ranged and not any(key in self.context for key in ("page", "page_size")):
|
||||
return scope, None, None
|
||||
page, page_size = self.context.get("page", 1), self.context.get("page_size", 100)
|
||||
if type(page) is not int or page < 1:
|
||||
raise ValueError("page must be an integer greater than or equal to 1")
|
||||
if type(page_size) is not int or not 1 <= page_size <= 1000:
|
||||
raise ValueError("page_size must be an integer between 1 and 1000")
|
||||
return scope, page, page_size
|
||||
|
||||
@classmethod
|
||||
def _scan(cls, workspace_dir: Path, daily_dir: str, scope: dict, allowed_paths) -> list[dict]:
|
||||
"""Inspect existing date directories only; permission checks precede file reads."""
|
||||
root, err = resolve_path(workspace_dir, daily_dir)
|
||||
if err or root is None:
|
||||
raise ValueError(err or "invalid daily directory")
|
||||
if "date" in scope:
|
||||
days = [scope["date"]]
|
||||
elif root.is_dir():
|
||||
days = sorted(
|
||||
(
|
||||
entry.name
|
||||
for entry in root.iterdir()
|
||||
if parse_daily_date(entry.name) == entry.name
|
||||
and scope["start_date"] <= entry.name <= scope["end_date"]
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
else:
|
||||
days = []
|
||||
return [
|
||||
cls._project(note)
|
||||
for day in days
|
||||
for note in scan_notes(workspace_dir, day, daily_dir, allowed_paths=allowed_paths)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _project(note: dict) -> dict:
|
||||
|
|
@ -56,19 +100,46 @@ class DailyListStep(BaseStep):
|
|||
return " ".join(parts)
|
||||
|
||||
async def execute(self):
|
||||
"""Scan ``<daily_dir>/<date>/`` and emit one projected record per note."""
|
||||
"""List authorized notes, preserving the legacy single-day response by default."""
|
||||
assert self.context is not None
|
||||
day, daily_dir, workspace_dir = self._collect_params()
|
||||
_target_dir, err = resolve_path(workspace_dir, f"{daily_dir}/{day}")
|
||||
if err:
|
||||
try:
|
||||
scope, page, page_size = self._collect_params()
|
||||
workspace_dir = Path(self.file_store.workspace_path or ".").resolve()
|
||||
notes = await asyncio.to_thread(
|
||||
self._scan,
|
||||
workspace_dir,
|
||||
self.config_value("daily_dir"),
|
||||
scope,
|
||||
self.context.get("_allowed_paths"),
|
||||
)
|
||||
except (ValueError, OSError, RuntimeError) as exc:
|
||||
self.context.response.success = False
|
||||
self.context.response.answer = f"Error: {err}"
|
||||
self.context.response.metadata.update({"date": day, "error": err})
|
||||
self.logger.info(f"[{self.name}] date={day} error={err!r}")
|
||||
self.context.response.answer = f"Error: {exc}"
|
||||
self.context.response.metadata["error"] = str(exc)
|
||||
return
|
||||
notes = [self._project(n) for n in scan_notes(workspace_dir, day, daily_dir)]
|
||||
metadata = dict(scope)
|
||||
if page is not None and page_size is not None:
|
||||
total = len(notes)
|
||||
metadata.update(
|
||||
{
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
"total_notes": total,
|
||||
"total_pages": (total + page_size - 1) // page_size,
|
||||
},
|
||||
)
|
||||
notes = notes[(page - 1) * page_size : page * page_size]
|
||||
metadata.update({"count": len(notes), "notes": notes})
|
||||
self.context.response.success = True
|
||||
lines = [self._format_note_line(n) for n in notes]
|
||||
self.context.response.answer = "\n".join(lines) if lines else f"No notes found for {day}"
|
||||
self.context.response.metadata.update({"date": day, "count": len(notes), "notes": notes})
|
||||
self.logger.info(f"[{self.name}] date={day} notes={len(notes)}")
|
||||
lines = [self._format_note_line(note) for note in notes]
|
||||
label = scope.get("date") or f"{scope['start_date']} to {scope['end_date']}"
|
||||
empty = f"No notes found for {label}"
|
||||
if metadata.get("total_notes") and not notes:
|
||||
empty = f"No notes on page {page} (total notes: {metadata['total_notes']})"
|
||||
answer = "\n".join(lines) if lines else empty
|
||||
if page is not None:
|
||||
header = " ".join(f"{key}: {metadata[key]}" for key in ("page", "page_size", "total_notes", "total_pages"))
|
||||
answer = f"{header}\n{answer}"
|
||||
self.context.response.answer = answer
|
||||
self.context.response.metadata.update(metadata)
|
||||
self.logger.info(f"[{self.name}] scope={scope} notes={len(notes)}")
|
||||
|
|
|
|||
318
tests/unit/test_daily_list_range.py
Normal file
318
tests/unit/test_daily_list_range.py
Normal file
|
|
@ -0,0 +1,318 @@
|
|||
"""Date-range and pagination contracts for the read-only daily_list job."""
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from reme.components import ApplicationContext
|
||||
from reme.components.file_store import LocalFileStore
|
||||
from reme.steps.file_io.daily_list import DailyListStep
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(name="step")
|
||||
async def daily_step(tmp_path, monkeypatch):
|
||||
"""Use a real file store in an isolated workspace, without model services."""
|
||||
monkeypatch.chdir(tmp_path)
|
||||
store = LocalFileStore(name="daily", embedding_store="")
|
||||
await store.start()
|
||||
context = ApplicationContext(workspace_dir=str(tmp_path), timezone="UTC")
|
||||
try:
|
||||
yield DailyListStep(file_store=store, app_context=context)
|
||||
finally:
|
||||
await store.close()
|
||||
|
||||
|
||||
def note(root: Path, day: str, name: str, *, daily_dir="daily", metadata="") -> Path:
|
||||
"""Create a note whose full frontmatter must survive projection."""
|
||||
path = root / daily_dir / day / f"{name}.md"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(f"---\nname: {name}\n{metadata}---\nbody\n", encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_range_is_inclusive_newest_first_and_keeps_full_frontmatter(step, tmp_path):
|
||||
"""Range boundaries and page order must not overwrite user metadata."""
|
||||
for day, name in [("2024-02-28", "old"), ("2024-02-29", "b"), ("2024-02-29", "a"), ("2024-03-01", "new")]:
|
||||
note(tmp_path, day, name, metadata="date: custom\nlabels: [one, two]\n")
|
||||
await step(start_date="2024-02-29", end_date="2024-03-01", page_size=2)
|
||||
response = step.context.response
|
||||
assert response.success
|
||||
assert response.metadata == {
|
||||
"start_date": "2024-02-29",
|
||||
"end_date": "2024-03-01",
|
||||
"page": 1,
|
||||
"page_size": 2,
|
||||
"total_notes": 3,
|
||||
"total_pages": 2,
|
||||
"count": 2,
|
||||
"notes": [
|
||||
{"path": "daily/2024-03-01/new.md", "name": "new", "date": "custom", "labels": ["one", "two"]},
|
||||
{"path": "daily/2024-02-29/a.md", "name": "a", "date": "custom", "labels": ["one", "two"]},
|
||||
],
|
||||
}
|
||||
assert "daily/2024-03-01/new.md" in response.answer
|
||||
assert "daily/2024-02-29/b.md" not in response.answer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_opt_in_pagination_preserves_unpaged_single_day_shape(step, tmp_path):
|
||||
"""Pagination is optional, stable and explicit about pages beyond the end."""
|
||||
for name in ("a", "b", "c"):
|
||||
note(tmp_path, "2024-03-01", name)
|
||||
await step(date="2024-03-01")
|
||||
assert set(step.context.response.metadata) == {"date", "count", "notes"}
|
||||
assert step.context.response.metadata["count"] == 3
|
||||
await step(date="2024-03-01", page=2, page_size=2)
|
||||
result = step.context.response.metadata
|
||||
assert result["count"] == 1 and result["total_notes"] == 3
|
||||
assert result["notes"][0]["name"] == "c"
|
||||
await step(date="2024-03-01", page=3, page_size=2)
|
||||
assert step.context.response.success
|
||||
assert step.context.response.metadata["notes"] == []
|
||||
assert step.context.response.metadata["total_pages"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"params",
|
||||
[
|
||||
{"start_date": "2024-03-01"},
|
||||
{"end_date": "2024-03-01"},
|
||||
{"start_date": None, "end_date": "2024-03-01"},
|
||||
{"start_date": "", "end_date": "2024-03-01"},
|
||||
{"start_date": "2024-W09-4", "end_date": "2024-03-01"},
|
||||
{"start_date": "2024-03-02", "end_date": "2024-03-01"},
|
||||
{"start_date": "2024-02-30", "end_date": "2024-03-01"},
|
||||
{"start_date": "2024-03-01T00:00:00", "end_date": "2024-03-02"},
|
||||
{"date": "2024-03-01", "start_date": "2024-03-01", "end_date": "2024-03-02"},
|
||||
{"page": 0},
|
||||
{"page_size": 1001},
|
||||
{"page_size": 0},
|
||||
{"page": "2"},
|
||||
{"page": None},
|
||||
{"page": True},
|
||||
{"page_size": False},
|
||||
{"page_size": 1.5},
|
||||
],
|
||||
)
|
||||
async def test_invalid_requests_fail_instead_of_falling_back_to_today(step, params):
|
||||
"""Malformed or conflicting queries must not quietly list today's notes."""
|
||||
await step(**params)
|
||||
assert not step.context.response.success
|
||||
assert "Error:" in step.context.response.answer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_range_page_is_bounded_but_legacy_calls_return_every_note(step, tmp_path):
|
||||
"""Omitted pagination must not truncate existing single-day callers."""
|
||||
for index in range(101):
|
||||
note(tmp_path, "2024-03-01", f"note-{index:03}")
|
||||
await step(date="2024-03-01")
|
||||
assert step.context.response.metadata["count"] == 101
|
||||
assert set(step.context.response.metadata) == {"date", "count", "notes"}
|
||||
await step(start_date="2024-03-01", end_date="2024-03-01")
|
||||
result = step.context.response.metadata
|
||||
assert result["count"] == result["page_size"] == 100
|
||||
assert result["total_notes"] == 101 and result["total_pages"] == 2
|
||||
await step(start_date="2024-03-01", end_date="2024-03-01", page=2)
|
||||
assert step.context.response.metadata["notes"][0]["name"] == "note-100"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_range_is_read_only_and_has_no_pages(step, tmp_path):
|
||||
"""A query must not create missing date directories or indexes."""
|
||||
await step(start_date="0001-01-01", end_date="9999-12-31")
|
||||
assert step.context.response.success
|
||||
assert step.context.response.metadata["total_notes"] == 0
|
||||
assert step.context.response.metadata["total_pages"] == 0
|
||||
assert step.context.response.metadata["notes"] == []
|
||||
assert not (tmp_path / "daily").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_range_uses_custom_directory_and_skips_indexes_invalid_dates_and_nested_files(step, tmp_path):
|
||||
"""Scan existing valid date directories, never every date or nested content."""
|
||||
step.app_context.app_config.daily_dir = "journal/days"
|
||||
note(tmp_path, "2024-03-01", "kept", daily_dir="journal/days")
|
||||
note(tmp_path, "not-a-date", "ignored", daily_dir="journal/days")
|
||||
note(tmp_path, "2024-W09-5", "week-date", daily_dir="journal/days")
|
||||
note(tmp_path, "2024-03-01/nested", "ignored", daily_dir="journal/days")
|
||||
note(tmp_path, "2024-03-01", "other-root")
|
||||
index = tmp_path / "journal/days/2024-03-01.md"
|
||||
index.write_text("day index", encoding="utf-8")
|
||||
(tmp_path / "journal/days/2024-03-02").write_text("not a directory", encoding="utf-8")
|
||||
await step(start_date="0001-01-01", end_date="9999-12-31")
|
||||
assert step.context.response.metadata["notes"] == [{"name": "kept", "path": "journal/days/2024-03-01/kept.md"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_permissions_filter_before_reading_and_before_counting(step, tmp_path, monkeypatch):
|
||||
"""An exact-file scope must hide other dates and sibling frontmatter."""
|
||||
allowed = note(tmp_path, "2024-03-01", "allowed")
|
||||
denied = {note(tmp_path, "2024-03-01", "sibling"), note(tmp_path, "2024-02-29", "other-day")}
|
||||
original = Path.read_text
|
||||
|
||||
def guarded_read(path, *args, **kwargs):
|
||||
if path in denied:
|
||||
pytest.fail("permission-denied note was read")
|
||||
return original(path, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(Path, "read_text", guarded_read)
|
||||
for params in ({"date": "2024-03-01"}, {"start_date": "2024-02-29", "end_date": "2024-03-01"}):
|
||||
await step(**params, _allowed_paths=[str(allowed.relative_to(tmp_path))])
|
||||
result = step.context.response.metadata
|
||||
assert result["count"] == 1
|
||||
assert result["notes"] == [{"name": "allowed", "path": "daily/2024-03-01/allowed.md"}]
|
||||
assert result.get("total_notes", 1) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("scope", [[], {}, 42, ["../outside"], ["daily", "../outside"]])
|
||||
async def test_invalid_permission_constraints_fail_closed(step, tmp_path, scope):
|
||||
"""An invalid scope cannot become an unrestricted listing."""
|
||||
note(tmp_path, "2024-03-01", "hidden")
|
||||
await step(start_date="2024-03-01", end_date="2024-03-01", _allowed_paths=scope)
|
||||
assert step.context.response.metadata["notes"] == []
|
||||
assert step.context.response.metadata["total_notes"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_directory_scope_counts_only_authorized_notes(step, tmp_path):
|
||||
"""Filtering happens before pagination and totals, including directory scopes."""
|
||||
note(tmp_path, "2024-03-01", "a")
|
||||
note(tmp_path, "2024-03-01", "b")
|
||||
note(tmp_path, "2024-02-29", "hidden")
|
||||
await step(start_date="2024-02-29", end_date="2024-03-01", page_size=1, _allowed_paths=["daily/2024-03-01"])
|
||||
result = step.context.response.metadata
|
||||
assert result["total_notes"] == result["total_pages"] == 2
|
||||
assert result["notes"][0]["name"] == "a"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_symlink_escapes_and_loops_do_not_leak_notes(step, tmp_path):
|
||||
"""Both date-directory links and note links retain workspace containment."""
|
||||
outside = tmp_path.parent / f"{tmp_path.name}-outside"
|
||||
outside.mkdir()
|
||||
(outside / "secret.md").write_text("---\nname: secret\n---\nprivate", encoding="utf-8")
|
||||
kept = note(tmp_path, "2024-03-01", "kept")
|
||||
try:
|
||||
(kept.parent / "escape.md").symlink_to(outside / "secret.md")
|
||||
(tmp_path / "daily/2024-02-29").symlink_to(outside, target_is_directory=True)
|
||||
(tmp_path / "daily/2024-03-02").symlink_to("2024-03-02", target_is_directory=True)
|
||||
except OSError as exc:
|
||||
pytest.skip(f"symlinks unavailable: {exc}")
|
||||
await step(start_date="2024-02-29", end_date="2024-03-02")
|
||||
assert step.context.response.metadata["notes"] == [{"name": "kept", "path": "daily/2024-03-01/kept.md"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queries_preserve_note_and_index_contents_and_mtimes(step, tmp_path):
|
||||
"""Listing never reindexes, rewrites frontmatter, or touches user files."""
|
||||
note(tmp_path, "2024-03-01", "a")
|
||||
note(tmp_path, "2024-02-29", "b")
|
||||
index = tmp_path / "daily/2024-03-01.md"
|
||||
index.write_text("handwritten index\n", encoding="utf-8")
|
||||
before = {p: (p.read_bytes(), p.stat().st_mtime_ns) for p in (tmp_path / "daily").rglob("*") if p.is_file()}
|
||||
await step(start_date="2024-02-29", end_date="2024-03-01", page_size=1)
|
||||
await step(date="2024-03-01", page=2, page_size=1)
|
||||
after = {p: (p.read_bytes(), p.stat().st_mtime_ns) for p in (tmp_path / "daily").rglob("*") if p.is_file()}
|
||||
assert before == after
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_or_unreadable_notes_do_not_count_toward_pages(step, tmp_path, monkeypatch):
|
||||
"""Match the existing scanner's skip behavior for malformed and unreadable notes."""
|
||||
good = note(tmp_path, "2024-03-01", "good")
|
||||
malformed = note(tmp_path, "2024-03-01", "bad")
|
||||
malformed.write_text("---\nlabels: [\n---\n", encoding="utf-8")
|
||||
unreadable = note(tmp_path, "2024-03-01", "unreadable")
|
||||
original = Path.read_text
|
||||
|
||||
def guarded_read(path, *args, **kwargs):
|
||||
if path == unreadable:
|
||||
raise PermissionError("unreadable note")
|
||||
return original(path, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(Path, "read_text", guarded_read)
|
||||
await step(start_date="2024-03-01", end_date="2024-03-01", page_size=1)
|
||||
assert step.context.response.metadata["total_notes"] == 1
|
||||
assert step.context.response.metadata["notes"][0]["path"] == good.relative_to(tmp_path).as_posix()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scanning_stays_off_the_event_loop(step, tmp_path, monkeypatch):
|
||||
"""File parsing uses the event loop's managed executor, not the caller's thread."""
|
||||
from threading import current_thread
|
||||
import frontmatter
|
||||
|
||||
note(tmp_path, "2024-03-01", "a")
|
||||
original = frontmatter.loads
|
||||
caller = current_thread().ident
|
||||
threads = []
|
||||
|
||||
def capture_thread(*args, **kwargs):
|
||||
threads.append(current_thread().ident)
|
||||
return original(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(frontmatter, "loads", capture_thread)
|
||||
await step(start_date="2024-03-01", end_date="2024-03-01")
|
||||
assert threads and all(thread != caller for thread in threads)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registered_job_and_http_endpoint_use_the_updated_schema(step, tmp_path):
|
||||
"""Exercise the configured Job and HTTP boundary rather than only a Step helper."""
|
||||
import httpx
|
||||
import yaml
|
||||
|
||||
from reme.components.job import BaseJob
|
||||
from reme.components.service import HttpService, MCPService
|
||||
from reme.enumeration import ComponentEnum
|
||||
|
||||
default = yaml.safe_load((Path(__file__).parents[2] / "reme/config/default.yaml").read_text(encoding="utf-8"))
|
||||
spec = default["jobs"]["daily_list"]
|
||||
properties = spec["parameters"]["properties"]
|
||||
assert {"date", "start_date", "end_date", "page", "page_size"} == set(properties)
|
||||
assert "default" not in properties["page"] and "default" not in properties["page_size"]
|
||||
assert properties["page_size"]["maximum"] == 1000
|
||||
note(tmp_path, "2024-02-29", "old")
|
||||
note(tmp_path, "2024-03-01", "new")
|
||||
context = step.app_context
|
||||
context.components[ComponentEnum.FILE_STORE] = {"default": step.file_store}
|
||||
job = BaseJob(name="daily_list", app_context=context, **spec)
|
||||
await job.start()
|
||||
try:
|
||||
response = await job(start_date="2024-02-29", end_date="2024-03-01", page_size=1)
|
||||
assert response.success and response.metadata["notes"][0]["name"] == "new"
|
||||
service = HttpService(mcp_enabled=False, web_enabled=False)
|
||||
service.build_service(SimpleNamespace(config=context.app_config))
|
||||
service.add_job(job)
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.ASGITransport(app=service.service),
|
||||
base_url="http://test",
|
||||
) as client:
|
||||
response = await client.post(
|
||||
"/daily_list",
|
||||
json={"start_date": "2024-02-29", "end_date": "2024-03-01", "page_size": 1},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["metadata"]["total_notes"] == 2
|
||||
response = await client.post("/daily_list", json={"date": "2024-03-01"})
|
||||
assert set(response.json()["metadata"]) == {"date", "count", "notes"}
|
||||
response = await client.post("/daily_list", json={"start_date": "2024-03-01"})
|
||||
assert not response.json()["success"]
|
||||
mcp = MCPService()
|
||||
mcp.build_service(SimpleNamespace(config=context.app_config))
|
||||
mcp.add_job(job)
|
||||
tool = await mcp.service.get_tool("daily_list")
|
||||
result = await tool.run({"start_date": "2024-02-29", "end_date": "2024-03-01", "page_size": 1})
|
||||
assert "total_notes: 2" in str(result.content)
|
||||
assert "daily/2024-03-01/new.md" in str(result.content)
|
||||
assert "daily/2024-02-29/old.md" not in str(result.content)
|
||||
legacy = await tool.run({"date": "2024-03-01"})
|
||||
assert "total_notes:" not in str(legacy.content)
|
||||
finally:
|
||||
await job.close()
|
||||
Loading…
Add table
Reference in a new issue