fix(benchmark): validate cached native summary provenance

benchmark/compaction/README.md

@@ -270,7 +270,7 @@ python -m benchmark.compaction.suite_run --live \
   --output /path/outside/repository/evaluation
 
 # Generate Hermes compaction summaries under the small-budget setting.
-# An existing complete --native directory reuses all frozen native generations.
+# An existing --native directory reuses frozen generations only after source validation.
 python -m benchmark.compaction.suite_native --live \
   --corpus /path/outside/repository/corpus/corpus.json \
   --evaluation /path/outside/repository/evaluation \
@@ -292,8 +292,12 @@ python -m benchmark.compaction.suite_fusion --live \
 
 This reproduces the protocol; new agent calls may produce different trajectories. `suite_fusion` re-answers only
 H + Top 1–5, retaining the other eight arms, rather than re-answering all thirteen. To reuse the recorded H generations,
-also pass `--native-evaluation /path/to/pre-amendment/evaluation`. The runner checks source hashes and permits only the
-held-out `exam` to differ; the summary generator never receives that field.
+also pass `--native-evaluation /path/to/pre-amendment/evaluation` to either `suite_native` or `suite_fusion` when reusing
+its cache. Both runners require `generation/validation.json` to match the SHA-256 of the source file used for generation
+and all three summary repeats to be present. By default that source is the current evaluation's `source.json`;
+`--native-evaluation` explicitly identifies the original source. Only the held-out `exam` may differ from the current
+source; the summary generator never receives that field. Missing provenance, a hash mismatch or changed evidence is
+rejected before answering calls. Preserve rejected caches and use a new `--native` path to generate replacements.
 
 ### Frozen offline verification
 

benchmark/compaction/README_ZH.md

@@ -251,7 +251,7 @@ python -m benchmark.compaction.suite_run --live \
   --output /path/outside/repository/evaluation
 
 # 生成小预算设置下的 Hermes 上下文压缩摘要。
-# 若 --native 已是完整目录，则复用其中全部固定摘要。
+# 若 --native 已存在，先校验来源，再复用其中固定摘要。
 python -m benchmark.compaction.suite_native --live \
   --corpus /path/outside/repository/corpus/corpus.json \
   --evaluation /path/outside/repository/evaluation \
@@ -272,8 +272,13 @@ python -m benchmark.compaction.suite_fusion --live \
 ```
 
 这复现的是实验协议，新 agent 调用可能产生不同轨迹。`suite_fusion` 只重新回答 H + Top 1～5，保留其他八组，
-不会重新回答全部十三组。复用本次已有 H 时，额外传入 `--native-evaluation /path/to/pre-amendment/evaluation`；
-程序校验来源哈希，仅允许评分用的 `exam` 不同，摘要生成器不接收该字段。
+不会重新回答全部十三组。复用本次已有 H 缓存时，向 `suite_native` 或 `suite_fusion` 额外传入
+`--native-evaluation /path/to/pre-amendment/evaluation`。
+两个程序均要求 `generation/validation.json` 中的 SHA-256 与生成摘要时使用的来源文件一致，且三次摘要均完整。
+默认来源为当前评测的 `source.json`；`--native-evaluation` 显式指定原始来源。
+仅允许当前来源中的评分用 `exam` 不同，摘要生成器不接收该字段。
+缺少来源记录、哈希不匹配或证据发生变化时，程序会在回答调用前拒绝缓存。
+请保留被拒绝的缓存，使用新的 `--native` 路径重新生成。
 
 ### 冻结结果离线复核
 

benchmark/compaction/suite_fusion.py

@@ -14,7 +14,7 @@ from openai import AsyncOpenAI
 from benchmark.compaction.cases import render, write
 from benchmark.compaction.run import Experiment, compose, messages, top_n
 from benchmark.compaction.suite import grade
-from benchmark.compaction.suite_native import validate_base
+from benchmark.compaction.suite_native import load_native_summaries, validate_base
 from benchmark.compaction.suite_run import ARMS, load_cases, report
 
 
@@ -37,17 +37,8 @@ def prepare(args, cases):
         if saved["messages"] != case["messages"] or saved["exam"] != case["exam"] or saved["gold"] != case["gold"]:
             raise ValueError("Corpus or evaluation questions changed")
         generation = args.native / key / "generation"
-        validation = json.loads((generation / "validation.json").read_text())
         native_source = (args.native_evaluation or args.evaluation) / key / "source.json"
-        if validation["source_sha256"] != hashlib.sha256(native_source.read_bytes()).hexdigest():
-            raise ValueError("Native summary source mismatch")
-        native_saved = json.loads(native_source.read_text())
-        # The format amendment changed only the held-out exam, never native generator input.
-        if {k: v for k, v in native_saved.items() if k != "exam"} != {k: v for k, v in saved.items() if k != "exam"}:
-            raise ValueError("Native source evidence changed")
-        summaries = json.loads((generation / "summaries.json").read_text())
-        if len(summaries) != 3 or {r["repeat"] for r in summaries} != set(range(3)):
-            raise ValueError("Native generations are incomplete")
+        summaries = load_native_summaries(generation, source, native_source)
         for entry in summaries:
             i = entry["repeat"]
             native = index[key, "hermes_summary", i]

benchmark/compaction/suite_native.py

@@ -37,11 +37,36 @@ def validate_base(evaluation, corpus, cases):
     return rows
 
 
+def load_native_summaries(generation, source, native_source=None):
+    """Verify cached provenance and allow only an explicitly identified exam-only amendment."""
+    validation_path = generation / "validation.json"
+    summaries_path = generation / "summaries.json"
+    if not validation_path.is_file() or not summaries_path.is_file():
+        raise ValueError("Native generation set is incomplete; preserve it and use a new path")
+    native_source = native_source or source
+    validation = json.loads(validation_path.read_text())
+    if validation.get("source_sha256") != hashlib.sha256(native_source.read_bytes()).hexdigest():
+        raise ValueError("Native summary source mismatch")
+    saved = json.loads(source.read_text())
+    native_saved = json.loads(native_source.read_text())
+    # The held-out exam is excluded from native summary generation.
+    if {k: v for k, v in native_saved.items() if k != "exam"} != {k: v for k, v in saved.items() if k != "exam"}:
+        raise ValueError("Native source evidence changed")
+    summaries = json.loads(summaries_path.read_text())
+    if len(summaries) != 3 or {r["repeat"] for r in summaries} != set(range(3)):
+        raise ValueError("Native generations are incomplete")
+    return summaries
+
+
 async def prepare_native(args, cases, repo):
     """Prepare a new isolated native generation set or validate an existing frozen set."""
     if args.native.exists():
-        if not all((args.native / key / "generation/summaries.json").is_file() for key in cases):
-            raise ValueError("Native generation set is incomplete; preserve it and use a new path")
+        for key in cases:
+            load_native_summaries(
+                args.native / key / "generation",
+                args.evaluation / key / "source.json",
+                (args.native_evaluation / key / "source.json") if args.native_evaluation else None,
+            )
         return
     if not args.hermes or not args.hermes_python:
         raise ValueError("New native generations require --hermes and --hermes-python")
@@ -137,8 +162,11 @@ async def run(args):
     for key, case in cases.items():
         root = output / key
         root.mkdir()
-        generated = json.loads((args.native / key / "generation/summaries.json").read_text())
-        assert len(generated) == 3 and {r["repeat"] for r in generated} == set(range(3))
+        generated = load_native_summaries(
+            args.native / key / "generation",
+            args.evaluation / key / "source.json",
+            (args.native_evaluation / key / "source.json") if args.native_evaluation else None,
+        )
         for entry in generated:
             i = entry["repeat"]
             full = next(r for r in original if r["case"] == key and r["repeat"] == i and r["arm"] == "full")
@@ -229,6 +257,7 @@ def main():
         parser.add_argument("--" + name, type=Path, required=True)
     parser.add_argument("--hermes", type=Path)
     parser.add_argument("--hermes-python", type=Path)
+    parser.add_argument("--native-evaluation", type=Path)
     parser.add_argument("--live", action="store_true")
     args = parser.parse_args()
     if not args.live:

tests/unit/test_compaction_study.py

@@ -7,7 +7,7 @@ from argparse import Namespace
 import pytest
 
 from benchmark.compaction.cases import event_units, write
-from benchmark.compaction.suite_native import validate_base
+from benchmark.compaction.suite_native import prepare_native, validate_base
 from benchmark.compaction.suite_fusion import prepare
 from benchmark.compaction.suite import context, define, exam, grade
 from benchmark.compaction.suite_run import ARMS, load_cases, metrics
@@ -153,6 +153,75 @@ def test_native_rejects_incompatible_or_duplicate_base_panels(tmp_path):
         validate_base(tmp_path, corpus, cases)
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "change,error",
+    [
+        ("none", None),
+        ("hash", "source mismatch"),
+        ("missing_validation", "incomplete"),
+        ("missing_hash", "source mismatch"),
+        ("missing_summaries", "incomplete"),
+        ("duplicate_repeat", "incomplete"),
+        ("exam_implicit", "source mismatch"),
+        ("exam_explicit", None),
+        ("messages", "evidence changed"),
+        ("gold", "evidence changed"),
+        ("units", "evidence changed"),
+    ],
+)
+async def test_native_cache_provenance_before_provider_calls(tmp_path, monkeypatch, change, error):
+    """Reject stale caches without generating or answering, except explicit exam-only amendments."""
+    evaluation = tmp_path / "evaluation"
+    source = evaluation / "example" / "source.json"
+    source.parent.mkdir(parents=True)
+    original = {
+        "messages": [{"role": "user", "content": "original task"}],
+        "exam": "original exam",
+        "gold": {"answer": 42},
+        "units": [{"id": "u", "text": "original evidence"}],
+    }
+    write(source, original)
+    generation = tmp_path / "native" / "example" / "generation"
+    generation.mkdir(parents=True)
+    validation_path = generation / "validation.json"
+    summaries_path = generation / "summaries.json"
+    write(validation_path, {"source_sha256": hashlib.sha256(source.read_bytes()).hexdigest()})
+    write(summaries_path, [{"repeat": i, "summary": f"frozen {i}"} for i in range(3)])
+    args = Namespace(native=tmp_path / "native", evaluation=evaluation, native_evaluation=None)
+    if change == "hash":
+        write(validation_path, {"source_sha256": "different-source"})
+    elif change == "missing_validation":
+        validation_path.unlink()
+    elif change == "missing_hash":
+        write(validation_path, {})
+    elif change == "missing_summaries":
+        summaries_path.unlink()
+    elif change == "duplicate_repeat":
+        write(summaries_path, [{"repeat": 0}] * 3)
+    elif change in ("exam_implicit", "exam_explicit", "messages", "gold", "units"):
+        native_source = tmp_path / "original" / "example" / "source.json"
+        native_source.parent.mkdir(parents=True)
+        write(native_source, original)
+        changed = {**original, "exam": "amended exam"}
+        if change in ("messages", "gold", "units"):
+            changed[change] = []
+        write(source, changed)
+        if change != "exam_implicit":
+            args.native_evaluation = tmp_path / "original"
+
+    def no_provider_calls(*_args, **_kwargs):
+        pytest.fail("Invalid cached sources must be rejected before provider calls")
+
+    monkeypatch.setattr("benchmark.compaction.suite_native.asyncio.create_subprocess_exec", no_provider_calls)
+    monkeypatch.setattr("benchmark.compaction.suite_native.AsyncOpenAI", no_provider_calls)
+    if error:
+        with pytest.raises(ValueError, match=error):
+            await prepare_native(args, ["example"], tmp_path / "repo")
+    else:
+        await prepare_native(args, ["example"], tmp_path / "repo")
+
+
 def test_hermes_fusion_reuses_summary_and_verified_jev_prefix(tmp_path, monkeypatch):
     """Appending evidence cannot rewrite the frozen native summary or rerank units."""
     case = {