diff --git a/plugins/procedure-memory/README.md b/plugins/procedure-memory/README.md index 4463cfb3..1e53000d 100644 --- a/plugins/procedure-memory/README.md +++ b/plugins/procedure-memory/README.md @@ -81,7 +81,7 @@ The operation journal records the destination, target content and stable scoped Failed files remain intact and are temporarily excluded from **both** search views. Startup and unrelated trajectories continue. Search returns the available experiences with `index_complete=false` and a `pending_indexes` list containing the excluded paths, versions and errors, rather than silently returning stale records. A corrected file re-enters both views after synchronization. A segment with its own pending writes cannot advance; resume lets the agent run again to correct them. Structural index failures that prevent safe exclusion still fail explicitly. -Checkpoints save segment position, bounded summary, processing fingerprint, recoverable input and final reply. The fingerprint includes the segmentation algorithm version; finish runs created by the previous segmentation algorithm with that plugin version before upgrading. Incompatible checkpoints fail explicitly without changing source files or progress. Long lines use bounded, progressively larger tokenization windows instead of repeatedly encoding the remaining line. A resumed segment receives its prior operation ledger so it can read completed experiences and continue; it does not assume the model will repeat identical decisions. A saved final reply is reused if the segment checkpoint failed to advance. Restart reconciles pending writes and rebuilds both views from experience JSON. Rebuilds never delete or rewrite experience or trajectory files. +Checkpoints save segment position, bounded summary, processing fingerprint, recoverable input and final reply. The fingerprint includes the segmentation algorithm version; finish runs created by the previous segmentation algorithm with that plugin version before upgrading. Incompatible checkpoints fail explicitly without changing source files or progress. Segmentation predicts bounded candidates from measured token density, targets about 95% of the token budget and halves oversized candidates. Every emitted segment is checked exactly; whole-line boundaries are preferred in the latter half of a candidate, and records may be split. Segments are lossless but need not fill the budget. A resumed segment receives its prior operation ledger so it can read completed experiences and continue; it does not assume the model will repeat identical decisions. A saved final reply is reused if the segment checkpoint failed to advance. Restart reconciles pending writes and rebuilds both views from experience JSON. Rebuilds never delete or rewrite experience or trajectory files. For explicit maintenance, with other workspace writers stopped: diff --git a/plugins/procedure-memory/README_ZH.md b/plugins/procedure-memory/README_ZH.md index b82cf9f2..2d198afb 100644 --- a/plugins/procedure-memory/README_ZH.md +++ b/plugins/procedure-memory/README_ZH.md @@ -83,7 +83,7 @@ auto-memory 获得 `procedure_search`、`procedure_read`、`procedure_create`、 失败经验的 JSON 保留,并暂时从**两个**搜索视图排除;启动及其他轨迹仍可继续。检索返回可用经验,同时明确设置 `index_complete=false`,通过 `pending_indexes` 列出排除的路径、版本与错误,避免静默返回过期记录。修正并完成同步后重新进入两个视图。当前段若仍有待同步写入,不推进检查点;续跑会再次调用 agent 进行修正。若底层索引故障导致无法安全排除记录,仍明确报错。 -检查点保存段位置、摘要、配置指纹、提炼输入及最终响应。指纹包含分段算法版本;旧分段算法创建的未完成任务须先使用原插件版本完成,再升级。不兼容检查点明确报错,不改动原文件或进度。超长单行通过逐步扩大的局部窗口计算 token,不再反复编码剩余全文。续跑将已有操作记录交给 agent,以读取已写经验后继续;若最终响应已保存但段位置未推进,复用响应。启动时恢复未完成操作,并从经验 JSON 重建索引。维护操作不改写或删除原始轨迹和经验。 +检查点保存段位置、摘要、配置指纹、提炼输入及最终响应。指纹包含分段算法版本;旧分段算法创建的未完成任务须先使用原插件版本完成,再升级。不兼容检查点明确报错,不改动原文件或进度。分段根据实测 token 密度预测有界候选,目标为 token 上限的约 95%,超限时减半重试。每个输出段均经过精确计数;优先选择候选后半区的完整行边界,必要时拆分记录。文本无损,但不保证填满预算。续跑将已有操作记录交给 agent,以读取已写经验后继续;若最终响应已保存但段位置未推进,复用响应。启动时恢复未完成操作,并从经验 JSON 重建索引。维护操作不改写或删除原始轨迹和经验。 停止同 workspace 的其他写入进程后,可显式重建: diff --git a/plugins/procedure-memory/src/reme_procedure_memory/trajectory.py b/plugins/procedure-memory/src/reme_procedure_memory/trajectory.py index deb60b8e..0076c561 100644 --- a/plugins/procedure-memory/src/reme_procedure_memory/trajectory.py +++ b/plugins/procedure-memory/src/reme_procedure_memory/trajectory.py @@ -5,8 +5,6 @@ import json import os import shutil import tempfile -from bisect import bisect_right -from itertools import chain from pathlib import Path from reme.steps.file_io import validate_filename_component @@ -15,7 +13,9 @@ from reme.steps.file_io._file_io import detect_file_encoding from .checkpoint import atomic_json, checked_path, digest, load_json SUPPORTED = {"", ".txt", ".md", ".markdown", ".json", ".jsonl", ".log"} -SEGMENTATION_VERSION = 4 +SEGMENTATION_VERSION = 5 +_MAX_SEGMENT_CHARS = 1024 * 1024 +_DENSITY_SAMPLE_CHARS = 4096 def file_hash(path: Path, max_bytes: int) -> str: @@ -125,66 +125,64 @@ async def prepare_text(path: Path) -> str: return encoding +def _segment_end(text: str, start: int, length: int) -> int: + """Prefer a line ending in the latter half without splitting CRLF when possible.""" + end = start + length + if length > 1 and text[end - 1 : end + 1] == "\r\n": + end -= 1 + lower = start + (length - 1) // 2 + boundary = max(text.rfind("\n", lower, end), text.rfind("\r", lower, end)) + return boundary + 1 if boundary >= lower else end + + def segments(path: Path, encoding: str, budget): - """Batch exact checks while keeping line boundaries and lossless, bounded output.""" - parts = [] - boundaries = [] - size = 0 - offset = 0 + """Predict bounded candidates, check exactly once, and halve oversized candidates. + + Probes shrink geometrically; the next prediction grows by at most twice the + emitted length. Including one initial density sample, total counter input is + less than 8N characters, even for nonmonotonic counts. Segments need not fill + the budget. + """ limit = budget.limits["segment"] - check_at = limit - checked_size = 0 + target_tokens = limit - limit // 20 + target = min(limit, _MAX_SEGMENT_CHARS, _DENSITY_SAMPLE_CHARS) + calibrating = True + pending = "" + start = offset = 0 + eof = False with path.open(encoding=encoding, errors="strict", newline="") as stream: - for line in chain(stream, ("",)): - if line: - parts.append(line) - size += len(line) - boundaries.append(size) - # Check just below the predicted capacity so near-full batches - # can be emitted without searching for a maximal prefix. - if size < check_at - max(1, check_at // 100): - continue - elif not size: + while True: + # One character of lookahead keeps CRLF intact at window edges. + needed = target + 1 - (len(pending) - start) + if needed > 0 and not eof: + incoming = stream.read(needed) + pending = pending[start:] + incoming + start = 0 + eof = len(incoming) < needed + if start == len(pending): break - pending = "".join(parts) - if not line and size == checked_size: - yield {"offset": offset, "text": pending} - break - start = 0 - while start < len(pending): - remaining = len(pending) - start - if remaining <= check_at: - count = budget.count(pending[start:]) - if count <= limit: - if not line or count >= limit - max(1, limit // 100): - yield {"offset": offset, "text": pending[start:]} - offset += remaining - start = len(pending) - else: - # Predict the next batch from the exact count. The - # prediction only schedules checks, never accepts text. - check_at = max(limit, remaining * limit // max(1, count)) - break - end = start + budget.prefix(pending, limit, start=start) - if end == len(pending): - if not line: - yield {"offset": offset, "text": pending[start:]} - check_at = max(limit, remaining * 2) + length = min(target, len(pending) - start) + while True: + end = start + length if eof and length == len(pending) - start else _segment_end(pending, start, length) + text = pending[start:end] + count = budget.count(text) + if count <= limit: break - # Prefer the last complete record within the exact token bound. - # A record larger than the budget is split at the checked prefix. - boundary = bisect_right(boundaries, end) - if boundary and boundaries[boundary - 1] > start: - end = boundaries[boundary - 1] - # Token counts need not be additive or monotonic at newlines. - end = start + budget.prefix(pending[start:end], limit) - yield {"offset": offset, "text": pending[start:end]} - offset += end - start - start = end - parts = [pending[start:]] - boundaries = [value - start for value in boundaries if value > start] - size = len(pending) - start - checked_size = size - if not size: - check_at = limit + length = len(text) // 2 + if not length: + raise ValueError("Token budget cannot hold one character") + predicted = min(_MAX_SEGMENT_CHARS, max(1, len(text) * target_tokens // max(1, count))) + if calibrating: + calibrating = False + # Calibrate once before emitting so a sparse first sample does + # not become an unnecessary model invocation of its own. + if predicted > len(text) and (not eof or end < len(pending)): + target = predicted + continue + yield {"offset": offset, "text": text} + offset += len(text) + start = end + # Density is only a scheduling hint. Capping growth also bounds the + # cost of mispredictions when token density changes abruptly. + target = min(predicted, len(text) * 2) diff --git a/plugins/procedure-memory/tests/test_auto_memory.py b/plugins/procedure-memory/tests/test_auto_memory.py index f7acde70..7ad17373 100644 --- a/plugins/procedure-memory/tests/test_auto_memory.py +++ b/plugins/procedure-memory/tests/test_auto_memory.py @@ -12,8 +12,8 @@ import pytest async def test_default_long_segments_and_internal_summary_slack(app_factory, tmp_path): app, store = await app_factory() source = tmp_path / "long.log" - first_segment = "x" * (96 * 1024 - 1) + "\n" - source.write_text(first_segment + "last segment") + text = "x" * (96 * 1024 - 1) + "\nlast segment" + source.write_text(text) seen = [] async def reply(inputs, _options): @@ -28,7 +28,10 @@ async def test_default_long_segments_and_internal_summary_slack(app_factory, tmp app.context.metadata["fake_reply"] = reply result = await app.context.jobs["procedure_auto_memory"](file=str(source), filename="long.log", request_id="long") assert result.success, result.answer - assert [data["trajectory_text"] for data in seen] == [first_segment, "last segment"] + assert len(seen) == 2 + assert "".join(data["trajectory_text"] for data in seen) == text + assert all(0 < store.budget.count(data["trajectory_text"]) <= 98304 for data in seen) + assert [data["offset"] for data in seen] == [0, len(seen[0]["trajectory_text"])] assert store.budget.limits["summary"] == 8192 diff --git a/plugins/procedure-memory/tests/test_trajectory.py b/plugins/procedure-memory/tests/test_trajectory.py index 10ace791..ecd6ca79 100644 --- a/plugins/procedure-memory/tests/test_trajectory.py +++ b/plugins/procedure-memory/tests/test_trajectory.py @@ -4,6 +4,7 @@ import pytest +from reme_procedure_memory import trajectory from reme_procedure_memory.checkpoint import atomic_json, digest, load_json from reme_procedure_memory.token_budget import TokenBudget from reme_procedure_memory.trajectory import prepare_text, segments @@ -59,6 +60,83 @@ def test_short_line_tokenization_work_is_linear(tmp_path, line_count, chars_per_ assert counted < len(text) * 1.6 +def test_low_density_text_does_not_emit_the_calibration_sample(tmp_path): + text = ("x" * 78 + "\r\n") * 2500 + path = tmp_path / "low-density.log" + path.write_bytes(text.encode()) + budget = TokenBudget(counter=lambda value: (len(value) + 3) // 4) + assert list(segments(path, "utf-8", budget)) == [{"offset": 0, "text": text}] + + +def test_final_partial_line_does_not_create_an_extra_segment(tmp_path): + text = "x" * 100 + "\ntail" + path = tmp_path / "final.log" + path.write_bytes(text.encode()) + assert list(segments(path, "utf-8", TokenBudget(counter=len, segment=128))) == [{"offset": 0, "text": text}] + + +@pytest.mark.parametrize("limit", [64, 257]) +@pytest.mark.parametrize("mode", ["bytes", "density_changes", "nonmonotonic", "zero"]) +def test_segmentation_counter_work_is_bounded_even_when_predictions_fail(tmp_path, limit, mode): + counted = 0 + + def exact_count(value): + if mode == "density_changes": + return sum(32 if char == "!" else 1 for char in value) + if mode == "nonmonotonic": + return len(value) + (limit // 2 if value.endswith(("\r", "\n")) else 0) + return 0 if mode == "zero" else len(value.encode()) + + def count(value): + nonlocal counted + counted += len(value) + return exact_count(value) + + text = ("a\n" * 256 + "!" * 1024 + "你好🙂\r\n" * 128) * 3 + path = tmp_path / "unpredictable.log" + path.write_bytes(text.encode()) + chunks = list(segments(path, "utf-8", TokenBudget(counter=count, segment=limit))) + assert counted < 8 * len(text) + assert "".join(chunk["text"] for chunk in chunks) == text + offset = 0 + for chunk in chunks: + assert chunk["text"] + assert exact_count(chunk["text"]) <= limit + assert chunk["offset"] == offset + offset += len(chunk["text"]) + + +@pytest.mark.parametrize("window", [2, 7, 128]) +def test_candidate_window_is_bounded_for_large_budgets(tmp_path, monkeypatch, window): + monkeypatch.setattr(trajectory, "_MAX_SEGMENT_CHARS", window) + counted = 0 + + def count(value): + nonlocal counted + counted += len(value) + assert len(value) <= window + return len(value) + + text = "x" * 10000 + "\r\n" + "你好🙂\r\n" * 100 + path = tmp_path / "bounded-window.log" + path.write_bytes(text.encode()) + chunks = list(segments(path, "utf-8", TokenBudget(counter=count, segment=100000))) + assert "".join(chunk["text"] for chunk in chunks) == text + assert counted == len(text) + assert all( + not (first["text"].endswith("\r") and second["text"].startswith("\n")) + for first, second in zip(chunks, chunks[1:]) + ) + + +def test_segment_budget_must_hold_one_character(tmp_path): + path = tmp_path / "too-small.log" + path.write_text("🙂", encoding="utf-8") + budget = TokenBudget(counter=lambda value: len(value.encode()), segment=3) + with pytest.raises(ValueError, match="cannot hold one character"): + list(segments(path, "utf-8", budget)) + + @pytest.mark.parametrize("text", ["", "abc", "a\nb\nc\nd\n", "a\rb\rc\r", "a\r\nb\nc\rtail"]) def test_batched_segments_preserve_line_boundaries(tmp_path, text): path = tmp_path / "lines.log"