From c4d9f5807c709a69fafb514ca66dbcc1e3ea8c65 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Mon, 19 Oct 2026 17:51:09 +0800 Subject: [PATCH] fix(benchmark): validate cached native summary provenance --- benchmark/compaction/README.md | 10 ++-- benchmark/compaction/README_ZH.md | 11 +++-- benchmark/compaction/suite_fusion.py | 13 +---- benchmark/compaction/suite_native.py | 37 +++++++++++++-- tests/unit/test_compaction_study.py | 71 +++++++++++++++++++++++++++- 5 files changed, 120 insertions(+), 22 deletions(-) diff --git a/benchmark/compaction/README.md b/benchmark/compaction/README.md index ca95f813..6269be52 100644 --- a/benchmark/compaction/README.md +++ b/benchmark/compaction/README.md @@ -270,7 +270,7 @@ python -m benchmark.compaction.suite_run --live \ --output /path/outside/repository/evaluation # Generate Hermes compaction summaries under the small-budget setting. -# An existing complete --native directory reuses all frozen native generations. +# An existing --native directory reuses frozen generations only after source validation. python -m benchmark.compaction.suite_native --live \ --corpus /path/outside/repository/corpus/corpus.json \ --evaluation /path/outside/repository/evaluation \ @@ -292,8 +292,12 @@ python -m benchmark.compaction.suite_fusion --live \ This reproduces the protocol; new agent calls may produce different trajectories. `suite_fusion` re-answers only H + Top 1–5, retaining the other eight arms, rather than re-answering all thirteen. To reuse the recorded H generations, -also pass `--native-evaluation /path/to/pre-amendment/evaluation`. The runner checks source hashes and permits only the -held-out `exam` to differ; the summary generator never receives that field. +also pass `--native-evaluation /path/to/pre-amendment/evaluation` to either `suite_native` or `suite_fusion` when reusing +its cache. Both runners require `generation/validation.json` to match the SHA-256 of the source file used for generation +and all three summary repeats to be present. By default that source is the current evaluation's `source.json`; +`--native-evaluation` explicitly identifies the original source. Only the held-out `exam` may differ from the current +source; the summary generator never receives that field. Missing provenance, a hash mismatch or changed evidence is +rejected before answering calls. Preserve rejected caches and use a new `--native` path to generate replacements. ### Frozen offline verification diff --git a/benchmark/compaction/README_ZH.md b/benchmark/compaction/README_ZH.md index 36669255..6afb193a 100644 --- a/benchmark/compaction/README_ZH.md +++ b/benchmark/compaction/README_ZH.md @@ -251,7 +251,7 @@ python -m benchmark.compaction.suite_run --live \ --output /path/outside/repository/evaluation # 生成小预算设置下的 Hermes 上下文压缩摘要。 -# 若 --native 已是完整目录，则复用其中全部固定摘要。 +# 若 --native 已存在，先校验来源，再复用其中固定摘要。 python -m benchmark.compaction.suite_native --live \ --corpus /path/outside/repository/corpus/corpus.json \ --evaluation /path/outside/repository/evaluation \ @@ -272,8 +272,13 @@ python -m benchmark.compaction.suite_fusion --live \ ``` 这复现的是实验协议，新 agent 调用可能产生不同轨迹。`suite_fusion` 只重新回答 H + Top 1～5，保留其他八组， -不会重新回答全部十三组。复用本次已有 H 时，额外传入 `--native-evaluation /path/to/pre-amendment/evaluation`； -程序校验来源哈希，仅允许评分用的 `exam` 不同，摘要生成器不接收该字段。 +不会重新回答全部十三组。复用本次已有 H 缓存时，向 `suite_native` 或 `suite_fusion` 额外传入 +`--native-evaluation /path/to/pre-amendment/evaluation`。 +两个程序均要求 `generation/validation.json` 中的 SHA-256 与生成摘要时使用的来源文件一致，且三次摘要均完整。 +默认来源为当前评测的 `source.json`；`--native-evaluation` 显式指定原始来源。 +仅允许当前来源中的评分用 `exam` 不同，摘要生成器不接收该字段。 +缺少来源记录、哈希不匹配或证据发生变化时，程序会在回答调用前拒绝缓存。 +请保留被拒绝的缓存，使用新的 `--native` 路径重新生成。 ### 冻结结果离线复核 diff --git a/benchmark/compaction/suite_fusion.py b/benchmark/compaction/suite_fusion.py index a9738211..588cfa70 100644 --- a/benchmark/compaction/suite_fusion.py +++ b/benchmark/compaction/suite_fusion.py @@ -14,7 +14,7 @@ from openai import AsyncOpenAI from benchmark.compaction.cases import render, write from benchmark.compaction.run import Experiment, compose, messages, top_n from benchmark.compaction.suite import grade -from benchmark.compaction.suite_native import validate_base +from benchmark.compaction.suite_native import load_native_summaries, validate_base from benchmark.compaction.suite_run import ARMS, load_cases, report @@ -37,17 +37,8 @@ def prepare(args, cases): if saved["messages"] != case["messages"] or saved["exam"] != case["exam"] or saved["gold"] != case["gold"]: raise ValueError("Corpus or evaluation questions changed") generation = args.native / key / "generation" - validation = json.loads((generation / "validation.json").read_text()) native_source = (args.native_evaluation or args.evaluation) / key / "source.json" - if validation["source_sha256"] != hashlib.sha256(native_source.read_bytes()).hexdigest(): - raise ValueError("Native summary source mismatch") - native_saved = json.loads(native_source.read_text()) - # The format amendment changed only the held-out exam, never native generator input. - if {k: v for k, v in native_saved.items() if k != "exam"} != {k: v for k, v in saved.items() if k != "exam"}: - raise ValueError("Native source evidence changed") - summaries = json.loads((generation / "summaries.json").read_text()) - if len(summaries) != 3 or {r["repeat"] for r in summaries} != set(range(3)): - raise ValueError("Native generations are incomplete") + summaries = load_native_summaries(generation, source, native_source) for entry in summaries: i = entry["repeat"] native = index[key, "hermes_summary", i] diff --git a/benchmark/compaction/suite_native.py b/benchmark/compaction/suite_native.py index 6ac93930..73a3a8cf 100644 --- a/benchmark/compaction/suite_native.py +++ b/benchmark/compaction/suite_native.py @@ -37,11 +37,36 @@ def validate_base(evaluation, corpus, cases): return rows +def load_native_summaries(generation, source, native_source=None): + """Verify cached provenance and allow only an explicitly identified exam-only amendment.""" + validation_path = generation / "validation.json" + summaries_path = generation / "summaries.json" + if not validation_path.is_file() or not summaries_path.is_file(): + raise ValueError("Native generation set is incomplete; preserve it and use a new path") + native_source = native_source or source + validation = json.loads(validation_path.read_text()) + if validation.get("source_sha256") != hashlib.sha256(native_source.read_bytes()).hexdigest(): + raise ValueError("Native summary source mismatch") + saved = json.loads(source.read_text()) + native_saved = json.loads(native_source.read_text()) + # The held-out exam is excluded from native summary generation. + if {k: v for k, v in native_saved.items() if k != "exam"} != {k: v for k, v in saved.items() if k != "exam"}: + raise ValueError("Native source evidence changed") + summaries = json.loads(summaries_path.read_text()) + if len(summaries) != 3 or {r["repeat"] for r in summaries} != set(range(3)): + raise ValueError("Native generations are incomplete") + return summaries + + async def prepare_native(args, cases, repo): """Prepare a new isolated native generation set or validate an existing frozen set.""" if args.native.exists(): - if not all((args.native / key / "generation/summaries.json").is_file() for key in cases): - raise ValueError("Native generation set is incomplete; preserve it and use a new path") + for key in cases: + load_native_summaries( + args.native / key / "generation", + args.evaluation / key / "source.json", + (args.native_evaluation / key / "source.json") if args.native_evaluation else None, + ) return if not args.hermes or not args.hermes_python: raise ValueError("New native generations require --hermes and --hermes-python") @@ -137,8 +162,11 @@ async def run(args): for key, case in cases.items(): root = output / key root.mkdir() - generated = json.loads((args.native / key / "generation/summaries.json").read_text()) - assert len(generated) == 3 and {r["repeat"] for r in generated} == set(range(3)) + generated = load_native_summaries( + args.native / key / "generation", + args.evaluation / key / "source.json", + (args.native_evaluation / key / "source.json") if args.native_evaluation else None, + ) for entry in generated: i = entry["repeat"] full = next(r for r in original if r["case"] == key and r["repeat"] == i and r["arm"] == "full") @@ -229,6 +257,7 @@ def main(): parser.add_argument("--" + name, type=Path, required=True) parser.add_argument("--hermes", type=Path) parser.add_argument("--hermes-python", type=Path) + parser.add_argument("--native-evaluation", type=Path) parser.add_argument("--live", action="store_true") args = parser.parse_args() if not args.live: diff --git a/tests/unit/test_compaction_study.py b/tests/unit/test_compaction_study.py index 3002ba61..80012634 100644 --- a/tests/unit/test_compaction_study.py +++ b/tests/unit/test_compaction_study.py @@ -7,7 +7,7 @@ from argparse import Namespace import pytest from benchmark.compaction.cases import event_units, write -from benchmark.compaction.suite_native import validate_base +from benchmark.compaction.suite_native import prepare_native, validate_base from benchmark.compaction.suite_fusion import prepare from benchmark.compaction.suite import context, define, exam, grade from benchmark.compaction.suite_run import ARMS, load_cases, metrics @@ -153,6 +153,75 @@ def test_native_rejects_incompatible_or_duplicate_base_panels(tmp_path): validate_base(tmp_path, corpus, cases) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "change,error", + [ + ("none", None), + ("hash", "source mismatch"), + ("missing_validation", "incomplete"), + ("missing_hash", "source mismatch"), + ("missing_summaries", "incomplete"), + ("duplicate_repeat", "incomplete"), + ("exam_implicit", "source mismatch"), + ("exam_explicit", None), + ("messages", "evidence changed"), + ("gold", "evidence changed"), + ("units", "evidence changed"), + ], +) +async def test_native_cache_provenance_before_provider_calls(tmp_path, monkeypatch, change, error): + """Reject stale caches without generating or answering, except explicit exam-only amendments.""" + evaluation = tmp_path / "evaluation" + source = evaluation / "example" / "source.json" + source.parent.mkdir(parents=True) + original = { + "messages": [{"role": "user", "content": "original task"}], + "exam": "original exam", + "gold": {"answer": 42}, + "units": [{"id": "u", "text": "original evidence"}], + } + write(source, original) + generation = tmp_path / "native" / "example" / "generation" + generation.mkdir(parents=True) + validation_path = generation / "validation.json" + summaries_path = generation / "summaries.json" + write(validation_path, {"source_sha256": hashlib.sha256(source.read_bytes()).hexdigest()}) + write(summaries_path, [{"repeat": i, "summary": f"frozen {i}"} for i in range(3)]) + args = Namespace(native=tmp_path / "native", evaluation=evaluation, native_evaluation=None) + if change == "hash": + write(validation_path, {"source_sha256": "different-source"}) + elif change == "missing_validation": + validation_path.unlink() + elif change == "missing_hash": + write(validation_path, {}) + elif change == "missing_summaries": + summaries_path.unlink() + elif change == "duplicate_repeat": + write(summaries_path, [{"repeat": 0}] * 3) + elif change in ("exam_implicit", "exam_explicit", "messages", "gold", "units"): + native_source = tmp_path / "original" / "example" / "source.json" + native_source.parent.mkdir(parents=True) + write(native_source, original) + changed = {**original, "exam": "amended exam"} + if change in ("messages", "gold", "units"): + changed[change] = [] + write(source, changed) + if change != "exam_implicit": + args.native_evaluation = tmp_path / "original" + + def no_provider_calls(*_args, **_kwargs): + pytest.fail("Invalid cached sources must be rejected before provider calls") + + monkeypatch.setattr("benchmark.compaction.suite_native.asyncio.create_subprocess_exec", no_provider_calls) + monkeypatch.setattr("benchmark.compaction.suite_native.AsyncOpenAI", no_provider_calls) + if error: + with pytest.raises(ValueError, match=error): + await prepare_native(args, ["example"], tmp_path / "repo") + else: + await prepare_native(args, ["example"], tmp_path / "repo") + + def test_hermes_fusion_reuses_summary_and_verified_jev_prefix(tmp_path, monkeypatch): """Appending evidence cannot rewrite the frozen native summary or rerank units.""" case = {