refactor(home): align evaluation summaries with dev_meta

This commit is contained in:
酱牛肉 2026-10-28 17:57:23 +08:00
parent f9918168ac
commit 95f9ea7b61
8 changed files with 877 additions and 155 deletions

View file

@ -241,24 +241,25 @@ Results use `home/evaluations/<branch>/<commit-prefix>/<ascending-run-id>/`:
manifest.json
summary.json
cases/<case-id>/
execution.json
memory_construction/
result.json
build.log
reme_workspace.tar.gz
reme_workspace/
evaluation.tar.gz
queries/
answers.json
result.json
<query-id>/answer.log
```
The memory tar is exported **before** queries, retains session Git checkpoints,
and is independent of the post-query evaluation archive. Replayed runs keep an
and is independent of the query execution state. Replayed runs keep an
independent donor tar and inherited build log without charging construction
tokens again. Answers are persisted before host judging. Per-query results keep
reference/actual answers, scores, judge output, token usage and failures. Summary
reference/actual answers, scores, compact judge output, reported answer token usage
and failures. `queries/result.json` is the sole authoritative answer record;
reconstructable prompts and repeated payloads are removed using the reviewed
`dev_meta` rules. Construction results similarly compact redundant links, Job
counts, token records and cumulative daily-index snapshots. Summary
scores use the expected-query denominator; infrastructure/judge/artifact errors
make the comparable mean unavailable while retaining the observed-score mean.
The Job returns `success=false` for infrastructure/judge/artifact failures, while
@ -273,8 +274,17 @@ their error text. Successful jobs returning empty answers still count as candida
failures. The construction failure-rate tolerance applies only to candidate
errors: infrastructure or unclassified failures always reject construction, even
when another failed job was a candidate error.
`valid_for_comparison` does not assert that an initial baseline covers the full
dataset: the caller must also verify its code and complete case/query coverage.
The manifest retains run identity, selection, mode and execution status. The
summary retains only outcome counts, expected/scored query counts, score means,
low-score cases, question-type statistics and compact case/query mappings.
Candidate answer token summaries average only total tokens per model; detailed
construction costs remain in construction results. Temporary post-query archives
are discarded after extracting logs; separate execution, checksum and raw-answer
JSON copies are not saved. See the
[artifact guide](src/reme_home/runtime_docs/validation-artifact-guide.md) for the
exact fields, rebuild helper and migration from the previous verbose format.
Before comparing runs, verify completed scored execution, infrastructure outcomes,
candidate code and complete case/query coverage.
Run/case/query `failure.json` files preserve diagnostics. Interrupted/fail-fast
runs publish `partial_summary.json`, not a completed summary. New executions
@ -292,8 +302,9 @@ returns `success=false` for candidate failures as well as infrastructure failure
It cannot reuse construction. Case/query selections refer to the selected mode's
dataset; omitted selections run all its cases/queries.
Debug results retain execution counts and artifacts, but all mean scores are
null and `valid_for_comparison=false`. `expected_scored_query_count` is zero,
while `expected_query_count` still records the expected execution count.
null. `scored_query_count` and `query_status_counts.unscored` are zero, while
`expected_query_count` still records the expected execution count. Debug/scored
mode is recorded in the manifest.
The former `dataset_kind` input is rejected; use `debug` instead. Debug results
must not be used to assess answer correctness or candidate quality. No real
Docker/model evaluation is required by Home's isolated unit tests.

View file

@ -7,6 +7,9 @@ from pathlib import Path, PurePosixPath
import shutil
import tarfile
import tempfile
from uuid import uuid4
from ..preparation.data.models import CaseSpec, QuerySpec
from ..workspace import _validate_directory
@ -32,13 +35,16 @@ def read_json(path: Path):
return json.loads(path.read_text(encoding="utf-8"))
def write_json(path: Path, value) -> None:
def write_json(path: Path, value, *, compact_leaf_containers: bool = False) -> None:
"""Atomically replace a rebuildable or progressively published result."""
path.parent.mkdir(parents=True, exist_ok=True)
descriptor, name = tempfile.mkstemp(prefix=".result-", dir=path.parent)
try:
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
json.dump(value, stream, ensure_ascii=False, allow_nan=False, indent=2)
if compact_leaf_containers:
stream.write(_dumps_compact_leaf_containers(value))
else:
json.dump(value, stream, ensure_ascii=False, allow_nan=False, indent=2)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
@ -47,6 +53,69 @@ def write_json(path: Path, value) -> None:
Path(name).unlink(missing_ok=True)
def _dumps_compact_leaf_containers(value) -> str:
"""Pretty-print JSON while keeping small scalar-only containers on one line."""
def is_atom(item) -> bool:
return isinstance(item, (str, int, float, bool, type(None)))
strings: set[str] = set()
def collect_strings(item) -> None:
if isinstance(item, str):
strings.add(item)
elif isinstance(item, dict):
for key, child in item.items():
if isinstance(key, str):
strings.add(key)
collect_strings(child)
elif isinstance(item, (list, tuple)):
for child in item:
collect_strings(child)
collect_strings(value)
marker_prefix = f"\0meta-reme-singleton-{uuid4().hex}-"
while any(item.startswith(marker_prefix) for item in strings):
marker_prefix = f"\0meta-reme-singleton-{uuid4().hex}-"
replacements: dict[str, str] = {}
def replace_leaf_containers(item):
if isinstance(item, dict):
if len(item) == 1 and is_atom(next(iter(item.values()))):
marker = f"{marker_prefix}{len(replacements)}"
replacements[marker] = json.dumps(
item,
ensure_ascii=False,
sort_keys=True,
allow_nan=False,
separators=(",", ": "),
)
return marker
return {key: replace_leaf_containers(child) for key, child in item.items()}
if isinstance(item, (list, tuple)):
if 1 <= len(item) <= 2 and all(is_atom(child) for child in item):
marker = f"{marker_prefix}{len(replacements)}"
replacements[marker] = json.dumps(
item,
ensure_ascii=False,
sort_keys=True,
allow_nan=False,
separators=(",", ": "),
)
return marker
children = [replace_leaf_containers(child) for child in item]
return children if isinstance(item, list) else tuple(children)
return item
transformed = replace_leaf_containers(value)
content = json.dumps(transformed, ensure_ascii=False, indent=2, sort_keys=True, allow_nan=False)
for marker, replacement in replacements.items():
encoded_marker = json.dumps(marker, ensure_ascii=False)
content = content.replace(encoded_marker, replacement)
return content
def file_sha256(path: Path) -> str:
"""Hash potentially large snapshots without loading them into memory."""
with path.open("rb") as stream:
@ -98,14 +167,19 @@ def average_tokens(queries: list[dict]) -> dict:
"""Average candidate answer total tokens over returned queries, like Meta-ReMe."""
totals = {}
for query in queries:
for model, usage in (query.get("answer_result") or {}).get("token_usage", {}).items():
usage_by_model = (query.get("answer_result") or {}).get("token_usage", {})
if not isinstance(usage_by_model, dict):
continue
for model, usage in usage_by_model.items():
if not isinstance(usage, dict):
continue
total = usage.get("total_tokens")
if isinstance(total, int) and not isinstance(total, bool):
if isinstance(total, int) and not isinstance(total, bool) and total != 0:
totals[model] = totals.get(model, 0) + total
return {model: round(total / len(queries), 3) for model, total in totals.items()} if queries else {}
def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = True) -> dict:
def summarize(run: Path, cases: list, *, scoring_required: bool = True) -> dict:
"""Rebuild compact optimizer evidence with Meta-ReMe's expected-query denominator."""
rows, all_scores, query_results = {}, [], []
case_counts = {"completed": 0, "candidate_failure": 0, "infra_error": 0}
@ -113,18 +187,19 @@ def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = T
query_counts = {"completed": 0, "candidate_failure": 0, "infra_error": 0}
stages = {"construction": 0, "query": 0}
expected = sum(len(case.queries) for case in cases)
expected_scored = expected if scoring_required else 0
for case in cases:
folder = run / "cases" / case.case_id
construction_path = folder / "memory_construction/result.json"
folder = contained(run, f"cases/{case.case_id}", directory=True)
construction_path = contained(folder, "memory_construction/result.json")
construction = read_json(construction_path) if construction_path.is_file() else {}
query_path = folder / "queries/result.json"
query_path = contained(folder, "queries/result.json")
queries = read_json(query_path).get("queries", []) if query_path.is_file() else []
failure = read_json(folder / "failure.json") if (folder / "failure.json").is_file() else {}
if not construction and not queries and not failure:
continue
scores = [query["score"] for query in queries if query.get("score") is not None]
all_scores.extend(scores)
query_results.extend(queries)
failed = sum(bool(query.get("error")) for query in queries)
failed = sum(query.get("error") is not None for query in queries)
has_infra = failure.get("failure_kind") == "infra_error" or any(q.get("infrastructure_error") for q in queries)
case_status = (
"infra_error"
@ -136,14 +211,16 @@ def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = T
if construction_status == "reused":
construction_status = "success"
construction_counts[construction_status] += 1
stages["construction"] += construction_status == "failure"
jobs = [job for job in construction.get("jobs", []) if isinstance(job, dict)]
job_results = [job["result"] if isinstance(job.get("result"), dict) else {} for job in jobs]
construction_errors = sum(not result.get("success") for result in job_results)
stages["construction"] += construction_status == "failure" or bool(construction_errors)
stages["query"] += bool(failed)
jobs = construction.get("jobs", [])
compact_queries = {}
for query in queries:
query_status = (
"completed"
if not query.get("error")
if query.get("error") is None
else "infra_error" if query.get("infrastructure_error") else "candidate_failure"
)
query_counts[query_status] += 1
@ -155,17 +232,19 @@ def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = T
rows[case.case_id] = {
"construction": {
"status": construction_status,
"success_job_count": sum(job.get("result", {}).get("success") is True for job in jobs),
"fail_job_count": sum(job.get("result", {}).get("success") is not True for job in jobs),
"success_job_count": len(jobs) - construction_errors,
"fail_job_count": construction_errors,
},
"expected_query_count": len(case.queries),
"expected_scored_query_count": len(case.queries) if scoring_required else 0,
"scored_count": len(scores),
"query_error_count": failed,
"mean_score": None if has_infra or not scoring_required else sum(scores) / len(case.queries),
"average_query_total_token_usage": average_tokens(queries),
"queries": compact_queries,
}
if failure.get("stage") is not None:
rows[case.case_id]["stage"] = failure["stage"]
if construction_errors:
rows[case.case_id]["construction_error_count"] = construction_errors
if failed:
rows[case.case_id]["query_error_count"] = failed
groups = {}
for query in query_results:
groups.setdefault(str(query.get("question_type") or "unknown"), []).append(query)
@ -178,25 +257,18 @@ def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = T
"average_query_total_token_usage": average_tokens(queries),
}
return {
"schema_version": 1,
"status": status,
"case_count": len(cases),
"expected_case_count": len(cases),
"case_count": len(rows),
"case_status_counts": case_counts,
"construction_status_counts": construction_counts,
"failure_stage_case_counts": stages,
"expected_query_count": expected,
"expected_scored_query_count": expected_scored,
"scoring_required": scoring_required,
"scored_count": len(all_scores),
"scored_query_count": len(all_scores),
"unscored_count": expected_scored - len(all_scores),
"query_status_counts": {**query_counts, "unscored": expected_scored - len(all_scores)},
"query_error_count": query_counts["candidate_failure"] + query_counts["infra_error"],
"scoring_complete": scoring_required and len(all_scores) == expected_scored,
"valid_for_comparison": scoring_required and status == "completed" and not case_counts["infra_error"],
"query_status_counts": {
**query_counts,
"unscored": max(expected - len(all_scores), 0) if scoring_required else 0,
},
"mean_query_score": (
sum(all_scores) / expected_scored if expected_scored and not case_counts["infra_error"] else None
sum(all_scores) / expected if scoring_required and expected and not case_counts["infra_error"] else None
),
"observed_mean_query_score": sum(all_scores) / len(all_scores) if all_scores else None,
"low_score_cases": [
@ -213,6 +285,38 @@ def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = T
}
def rebuild_summary(run: Path) -> dict:
"""Rebuild compact evidence from the manifest and authoritative phase results."""
manifest = read_json(run / "manifest.json")
cases = [
CaseSpec(
case_id=case_id,
sessions=[],
queries=[QuerySpec(query_id=query_id, question="", golden_answer=None) for query_id in query_ids],
)
for case_id, query_ids in manifest["query_ids"].items()
]
by_id = {case.case_id: case for case in cases}
if manifest.get("status") == "completed":
for case_id in manifest["case_ids"]:
folder = contained(run, f"cases/{case_id}", directory=True)
read_json(contained(folder, "memory_construction/result.json"))
query_path = contained(folder, "queries/result.json")
queries = read_json(query_path)["queries"] if query_path.is_file() else []
actual_ids = [query["query_id"] for query in queries]
if actual_ids != manifest["query_ids"][case_id]:
if queries or not (folder / "failure.json").is_file():
raise ValueError(f"Completed evaluation has incomplete query results: {case_id}")
summary = summarize(
run,
[by_id[case_id] for case_id in manifest["case_ids"]],
scoring_required=manifest.get("mode") != "debug",
)
name = "summary.json" if manifest.get("status") == "completed" else "partial_summary.json"
write_json(run / name, summary, compact_leaf_containers=True)
return summary
INFRA_TYPES = {
"APIConnectionError",
"APITimeoutError",

View file

@ -0,0 +1,177 @@
"""Result compaction rules preserved from dev_meta b53048e4.
Keep diagnostic evidence while removing reconstructable prompts and duplicate data.
Compaction copies inputs so scoring and failure classification retain raw responses.
"""
from collections.abc import Mapping
from copy import deepcopy
from pathlib import PurePosixPath
from typing import Any
def compact_query(result: dict) -> dict:
"""Copy and compact one authoritative query result."""
result = deepcopy(result)
_compact_query_result(result)
return result
def compact_construction(build: dict) -> dict:
"""Copy and compact ordered construction jobs using the reviewed rules."""
persisted_build = deepcopy(build)
_compact_construction_source_conversations(persisted_build)
_compact_construction_index_snapshots(persisted_build)
for job in persisted_build.get("jobs", []):
if not isinstance(job, Mapping):
continue
if isinstance(job, dict):
job.pop("memory_checkpoint", None)
result = job.get("result")
if not isinstance(result, dict):
continue
_compact_outer_job_call_count(job.get("job"), result)
token_usage = result.get("token_usage")
if not isinstance(token_usage, dict):
continue
for backend, usage in list(token_usage.items()):
if isinstance(usage, Mapping) and all(
usage.get(field) is None for field in ("input_tokens", "output_tokens", "total_tokens")
):
del token_usage[backend]
if not token_usage:
del result["token_usage"]
elif all(
isinstance(usage, Mapping)
and all(usage.get(field) == 0 for field in ("input_tokens", "output_tokens", "total_tokens"))
for usage in token_usage.values()
):
result["token_usage"] = 0
return persisted_build
def _compact_query_result(result: dict[str, Any]) -> None:
"""Remove reconstructable prompts and uninformative token records before persistence."""
answer_result = result.get("answer_result")
if isinstance(answer_result, dict):
if answer_result.get("answer") == result.get("answer"):
answer_result.pop("answer", None)
if "error" in result and answer_result.get("error") == result.get("error"):
answer_result.pop("error", None)
_compact_outer_job_call_count("agentic_answer", answer_result)
metadata = answer_result.get("metadata")
if isinstance(metadata, dict):
if metadata.get("query") == result.get("question"):
metadata.pop("query", None)
if metadata.get("response") == result.get("answer"):
metadata.pop("response", None)
judge_result = result.get("judge_result")
if isinstance(judge_result, dict):
judge_result.pop("token_usage", None)
if result.get("score") is not None:
judge_result.pop("answer", None)
_compact_outer_job_call_count("answer_judge", judge_result)
metadata = judge_result.get("metadata")
if isinstance(metadata, dict):
if metadata.get("raw_answer_judgement") is not None:
metadata.pop("answer_judgement", None)
for field in (
"agent_answer",
"golden_answer",
"llm_response",
"probing_question",
"query",
"question_type",
"rubric",
):
metadata.pop(field, None)
def compact(value: Any) -> None:
if isinstance(value, dict):
value.pop("sys_prompt", None)
token_usage = value.get("token_usage")
if isinstance(token_usage, dict):
for backend, usage in list(token_usage.items()):
if isinstance(usage, Mapping) and all(
usage.get(field) is None for field in ("input_tokens", "output_tokens", "total_tokens")
):
del token_usage[backend]
if not token_usage:
del value["token_usage"]
for child in value.values():
compact(child)
elif isinstance(value, list):
for child in value:
compact(child)
compact(result)
def _compact_outer_job_call_count(job_name: Any, result: dict[str, Any]) -> None:
"""Remove one call already represented by the enclosing result record."""
counts = result.get("job_call_counts")
if not isinstance(job_name, str) or not isinstance(counts, dict):
return
own_count = counts.get(job_name)
if isinstance(own_count, int) and not isinstance(own_count, bool) and own_count > 0:
remaining = own_count - 1
if remaining:
counts[job_name] = remaining
else:
del counts[job_name]
if not counts:
del result["job_call_counts"]
def _compact_construction_index_snapshots(value: Any) -> None:
"""Replace cumulative daily-index note snapshots with their counts."""
if isinstance(value, dict):
index = value.get("index")
if _is_daily_index_snapshot(index):
index["notes_count"] = len(index.pop("notes"))
for child in value.values():
_compact_construction_index_snapshots(child)
elif isinstance(value, list):
for child in value:
_compact_construction_index_snapshots(child)
def _is_daily_index_snapshot(value: Any) -> bool:
"""Recognize the stable ``refresh_day_index`` payload without assuming a configured directory name."""
if not isinstance(value, dict) or not isinstance(value.get("notes"), list):
return False
date = value.get("date")
path = value.get("path")
if not isinstance(date, str) or not isinstance(path, str):
return False
index_path = PurePosixPath(path)
return (
index_path.suffix == ".md"
and index_path.stem == date
and isinstance(value.get("created"), bool)
and isinstance(value.get("changed"), bool)
)
def _compact_construction_source_conversations(value: Any) -> None:
"""Drop a transcript link when its sibling session ID already identifies the same file."""
if isinstance(value, dict):
session_id = value.get("session_id")
source_conversation = value.get("source_conversation")
if isinstance(session_id, str) and isinstance(source_conversation, str):
source_path = source_conversation.strip()
if source_path.startswith("[[") and source_path[-2:] == "]]":
source_path = source_path[2:-2]
if PurePosixPath(source_path).name == f"{session_id}.jsonl":
del value["source_conversation"]
for child in value.values():
_compact_construction_source_conversations(child)
elif isinstance(value, list):
for child in value:
_compact_construction_source_conversations(child)

View file

@ -1,7 +1,6 @@
"""Home evaluation orchestration using the shared Docker pool and a host judge."""
import asyncio
from dataclasses import asdict
import os
import shutil
import traceback
@ -24,6 +23,7 @@ from .artifacts import (
failure_kind,
)
from .inputs import EvaluationSpec, load_inputs, reuse_inputs, snapshot_code
from .compaction import compact_construction, compact_query
from ..judge import HomeJudge
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
@ -146,9 +146,9 @@ class HomeEvaluation(BaseComponent):
"execution_mode": "query_only_replay" if source else "full_validation",
"construction_source": source,
"fingerprints": fingerprints,
"options": spec.model_dump(mode="json"),
"options": spec.model_dump(mode="json", exclude={"case_ids", "query_ids", "reuse_construction_from"}),
"judge": self.judge.identity(dataset.dataset),
"judge_model": self.judge.identity(dataset.dataset)["model"],
"status": "running",
"started_at": utc_now().isoformat(),
}
run = self._allocate(branch, commit, manifest)
@ -214,7 +214,15 @@ class HomeEvaluation(BaseComponent):
run / "cases" / plan.case_id / "failure.json",
{"stage": "artifacts", "error": result.error, "failure_kind": "infra_error"},
)
await self._score_case(run, by_id[plan.case_id], dataset.dataset, judge, semaphore, spec)
await self._score_case(
run,
by_id[plan.case_id],
dataset.dataset,
judge,
semaphore,
spec,
(result.queries or {}).get("queries", []),
)
results = await BenchmarkDockerPool(factory, concurrency=spec.concurrency).run(
plans,
@ -224,9 +232,6 @@ class HomeEvaluation(BaseComponent):
)
# Pool teardown can attach late cleanup failures after callbacks.
for result in results:
execution = asdict(result)
execution["workspace_snapshot"] = str(result.workspace_snapshot) if result.workspace_snapshot else None
write_json(run / "cases" / result.case_id / "execution.json", execution)
if result.error:
failure = run / "cases" / result.case_id / "failure.json"
if not failure.exists():
@ -236,22 +241,24 @@ class HomeEvaluation(BaseComponent):
if previous.get("error") != result.error:
previous.update(cleanup_error=result.error, failure_kind="infra_error")
write_json(failure, previous)
summary = summarize(run, cases, status="completed", scoring_required=not spec.debug)
summary.update(
execution_complete=True,
execution_mode=manifest["execution_mode"],
)
summary = summarize(run, cases, scoring_required=not spec.debug)
manifest.update(status="completed", finished_at=utc_now().isoformat())
write_json(run / "manifest.json", manifest)
write_json(run / "summary.json", summary)
write_json(run / "summary.json", summary, compact_leaf_containers=True)
return {
"result_dir": str(run),
"commit_sha": commit,
"execution_mode": manifest["execution_mode"],
"status": "completed",
**{key: value for key, value in summary.items() if key != "cases"},
}
except BaseException as error:
if run is not None:
manifest.update(
status="cancelled" if isinstance(error, asyncio.CancelledError) else "failed",
finished_at=utc_now().isoformat(),
)
write_json(run / "manifest.json", manifest)
write_json(
run / "failure.json",
{
@ -264,11 +271,9 @@ class HomeEvaluation(BaseComponent):
partial = summarize(
run,
cases,
status="cancelled" if isinstance(error, asyncio.CancelledError) else "failed",
scoring_required=not spec.debug,
)
partial["execution_complete"] = False
write_json(run / "partial_summary.json", partial)
write_json(run / "partial_summary.json", partial, compact_leaf_containers=True)
if isinstance(error, asyncio.CancelledError) or run is None:
raise
raise RuntimeError(f"Evaluation failed; artifacts: {run}: {error}") from error
@ -277,10 +282,25 @@ class HomeEvaluation(BaseComponent):
async def _persist_case(run, case, result, source, spec):
folder = run / "cases" / case.case_id
memory = folder / "memory_construction"
execution = asdict(result)
execution["workspace_snapshot"] = str(result.workspace_snapshot) if result.workspace_snapshot else None
write_json(folder / "execution.json", execution)
write_json(folder / "queries/answers.json", result.queries or {"queries": []})
# Freeze every returned answer in the sole authoritative query file
# before extraction or judging can fail. Unjudged answers stay present.
by_id = {row.get("query_id"): row for row in (result.queries or {}).get("queries", [])}
rows = [
compact_query(
{
**by_id[query.query_id],
"query_id": query.query_id,
"question": query.question,
"golden_answer": query.golden_answer,
"question_type": query.metadata.get("question_type"),
"score": None,
"judge_result": None,
},
)
for query in case.queries
if query.query_id in by_id
]
write_json(folder / "queries/result.json", {"queries": rows}, compact_leaf_containers=True)
if result.error:
stage = result.failure_stage or "execution"
exception = result.exception
@ -309,7 +329,7 @@ class HomeEvaluation(BaseComponent):
archive = memory / "reme_workspace.tar.gz"
if archive.is_file():
construction["workspace_sha256"] = await blocking(file_sha256, archive)
write_json(memory / "result.json", construction)
write_json(memory / "result.json", compact_construction(construction), compact_leaf_containers=True)
archive = memory / "reme_workspace.tar.gz"
if archive.is_file():
await blocking(inspect_archive, archive, spec.max_artifact_bytes, memory / "reme_workspace")
@ -325,25 +345,21 @@ class HomeEvaluation(BaseComponent):
target = folder / "queries" / query.query_id / "answer.log"
target.parent.mkdir(parents=True, exist_ok=True)
await blocking(shutil.copyfile, log, target)
# Do not merge the container's queries tree: it could overwrite
# host-frozen answers.json or host-only judge results. The complete
# untrusted tree remains available in evaluation.tar.gz for inspection.
# Container results cannot overwrite host-frozen answers or scores.
await blocking(shutil.rmtree, extracted)
integrity = {}
for relative in ("memory_construction/reme_workspace.tar.gz", "evaluation.tar.gz", "full.tar.gz"):
artifact = folder / relative
if artifact.is_file():
integrity[relative] = await blocking(file_sha256, artifact)
write_json(folder / "artifacts.json", {"sha256": integrity})
evaluation.unlink()
@staticmethod
async def _score_case(run, case, dataset, judge, semaphore, spec):
async def _score_case(run, case, dataset, judge, semaphore, spec, answers):
folder = run / "cases" / case.case_id
answers = read_json(folder / "queries/answers.json").get("queries", [])
by_id = {row.get("query_id"): row for row in answers}
rows = []
rows = read_json(folder / "queries/result.json").get("queries", [])
positions = {row["query_id"]: index for index, row in enumerate(rows)}
case_failure = read_json(folder / "failure.json") if (folder / "failure.json").exists() else {}
for query in case.queries:
raw = by_id.get(query.query_id, {})
if not raw and case_failure:
continue # Construction failures leave queries unassessed.
row = {
**raw,
"query_id": query.query_id,
@ -353,10 +369,9 @@ class HomeEvaluation(BaseComponent):
"score": None,
"judge_result": None,
}
if not isinstance(row.get("answer"), str) or not row["answer"].strip() or row.get("error"):
if not isinstance(row.get("answer"), str) or not row["answer"].strip() or row.get("error") is not None:
row["error"] = row.get("error") or "No successful answer (construction or execution failed)"
row["failure_stage"] = "answer"
case_failure = read_json(folder / "failure.json") if (folder / "failure.json").exists() else {}
row["failure_kind"] = (
failure_kind(row["exception"], "answer")
if row.get("exception")
@ -376,10 +391,14 @@ class HomeEvaluation(BaseComponent):
failure_kind="infra_error",
infrastructure_error=True,
)
rows.append(row)
compact = compact_query(row)
if query.query_id in positions:
rows[positions[query.query_id]] = compact
else:
rows.append(compact)
# Publish each score; a later judge failure cannot erase frozen answers.
write_json(folder / "queries/result.json", {"queries": rows})
if row.get("error"):
write_json(folder / "queries" / query.query_id / "failure.json", row)
write_json(folder / "queries/result.json", {"queries": rows}, compact_leaf_containers=True)
if row.get("error") is not None:
write_json(folder / "queries" / query.query_id / "failure.json", compact, compact_leaf_containers=True)
if spec.fail_fast:
raise RuntimeError(f"Query {case.case_id}/{query.query_id} failed: {row['error']}")

View file

@ -213,7 +213,7 @@ def reuse_inputs(home: Path, spec: EvaluationSpec, cases: list, fingerprints: di
raise ValueError("reuse_construction_from must be inside home/evaluations")
manifest = read_json(source / "manifest.json")
summary = read_json(source / "summary.json")
if manifest.get("dataset_kind") != "training-set" or summary.get("status") != "completed":
if manifest.get("dataset_kind") != "training-set" or manifest.get("status", summary.get("status")) != "completed":
raise ValueError("Reuse requires a completed training-set evaluation")
for key in ("dataset", "construction_config"):
if manifest.get("fingerprints", {}).get(key) != fingerprints[key]:

View file

@ -4,6 +4,7 @@
`home/evaluations/<branch>/<commit-prefix>/<run-id>/` inside the outer ReMe
workspace. Commit prefixes start at seven characters and extend on collision.
Run IDs increase within that commit directory; prior results are preserved.
Result compaction and summary fields follow the reviewed `dev_meta` format.
## Saved files
@ -11,65 +12,111 @@ Paths in this table are relative to one run directory:
| Path | Meaning |
| --- | --- |
| `manifest.json` | Candidate commit, case/query selection, execution mode, input/configuration fingerprints, host judge identity, and run status. |
| `summary.json` | Completed run summary: aggregate scores, expected/scored counts, case/query outcomes, low-score cases, and comparison validity. |
| `partial_summary.json` | Partial results after an interrupted or failed run; not a completed baseline. |
| `manifest.json` | Candidate commit, selected case/query IDs, dataset, execution mode, input/configuration fingerprints, host judge identity, options, timestamps and run status. Retained for reproducibility and rebuilding summaries; identity is not repeated in the summary. |
| `summary.json` | Rebuildable evidence from a completed execution. Its fields are described below. |
| `partial_summary.json` | Available evidence after cancellation or failure; not a completed baseline. |
| `failure.json` | Run-level failure details, when present. |
| `cases/<case-id>/execution.json` | Docker pool execution outcome, including execution/cleanup failures. |
| `cases/<case-id>/artifacts.json` | Checksums of exported archives. |
| `cases/<case-id>/memory_construction/result.json` | Construction outcome and ordered Job results, including available errors, metadata, and token usage. |
| `cases/<case-id>/memory_construction/build.log` | Construction log. Replayed runs inherit the donor log. |
| `cases/<case-id>/memory_construction/reme_workspace.tar.gz` | Workspace snapshot exported before answering queries; the donor for construction reuse. |
| `cases/<case-id>/memory_construction/result.json` | Ordered construction Job results with success, errors, metadata, answers, reported token usage and inner Job call counts. Replays record their source and snapshot checksum without charging construction again. |
| `cases/<case-id>/memory_construction/build.log` | Construction log. Replays inherit the donor log. |
| `cases/<case-id>/memory_construction/reme_workspace.tar.gz` | Checksum-verified workspace snapshot exported before answering; the donor for construction reuse. |
| `cases/<case-id>/memory_construction/reme_workspace/` | Extracted construction workspace, including memory files and session Git checkpoints when exported. |
| `cases/<case-id>/evaluation.tar.gz` | Exported post-query evaluation artifacts. |
| `cases/<case-id>/queries/answers.json` | Raw answers saved by the host before judging and artifact extraction. |
| `cases/<case-id>/queries/result.json` | Per-query question, reference and actual answer, score, answer response, judge response, and failure information. |
| `cases/<case-id>/queries/result.json` | The sole authoritative query record: question, reference and actual answer, question type, score, compact answer/judge responses and failures. All returned answers are frozen here before extraction and host judging; scores are published progressively. |
| `cases/<case-id>/queries/<query-id>/answer.log` | Query execution log, when available. |
| Case/query `failure.json` | Failure stage and diagnostics, when present. |
| `cases/<case-id>/full.tar.gz` | Best-effort sandbox diagnostics for execution failures, when available. |
Exported files depend on how far execution progressed. Infrastructure failures
and timeouts may prevent logs or full workspace archives from being recovered.
Inspect failure records before assuming that an absent artifact means the
candidate produced no memory or answer.
Successful exports retain query logs, then discard the temporary post-query
`evaluation.tar.gz`. There is no separate `execution.json`, `artifacts.json`,
`queries/answers.json` or per-query `result.json`. A failed extraction can leave
its archive for diagnosis. Exported files depend on execution progress; inspect
failure records before interpreting an absent workspace or log.
Query compaction removes reconstructable `sys_prompt` values, duplicate nested
answers/questions/errors, the enclosing Job's one call, judge token usage,
duplicate parsed judgements when raw judgements exist, and repeated judge inputs.
Distinct errors, raw judgements, rubric assessments and reported answer token
metrics remain. Unreported token backends are omitted.
Construction compaction removes checkpoint descriptors already available in Git,
one enclosing Job call, unreported token backends, and transcript links identified
by a sibling session ID. Recognized cumulative daily-index `notes` snapshots become
`notes_count`; unrelated plugin payloads remain. All-zero construction token
records become `0`. Detailed construction costs stay in this case artifact.
Small scalar containers are written on one line, following `dev_meta` formatting.
## Summary fields
The run summary contains only:
- `case_count`, `case_status_counts`, `construction_status_counts`, and
`failure_stage_case_counts` (construction/query; tolerated construction failures
still count as construction errors).
- `expected_query_count`, `scored_query_count`, and `query_status_counts`
(completed, candidate failure, infrastructure error, unscored).
- `mean_query_score`, `observed_mean_query_score`, `low_score_cases`,
`question_type_statistics`, and `cases`.
Each `cases[case_id]` contains construction `status`, `success_job_count` and
`fail_job_count`; `mean_score`; `average_query_total_token_usage`; and a `queries`
map. Each query entry contains only `question_type`, `score` and `status`.
`stage`, `construction_error_count` and `query_error_count` appear only when
applicable. There is no redundant case status or per-case coverage accounting.
Questions, answers, errors, Job details and construction token usage stay in
phase artifacts, outside the summary.
Token summaries contain only average candidate answer **total** tokens per model,
divided by all returned queries, including failures and queries without reported
usage. Zero and unknown token totals are omitted. Judge/construction costs and
input/output token breakdowns are not aggregated. Question-type statistics contain
query count, average of available scores (rounded to three decimals), and the same
answer token averages across cases. Missing/empty types form `unknown`.
Candidate failures count as zero in the expected-query score denominator.
Construction failures leave queries unassessed rather than inventing query
failures. Infrastructure, artifact, cleanup or judge errors make the comparable
run mean unavailable; `observed_mean_query_score` averages available scores only.
`low_score_cases` contains `{case_id: mean_score}` for cases with at least one
scored query below `1`, sorted by mean then case ID; unavailable case means are
omitted.
Run status and debug/scored mode are in the manifest. Debug runs skip judging;
run and case means are null, `scored_query_count` and the unscored count are zero,
while `expected_query_count` records execution coverage. They cannot measure
answer quality. Only compare completed scored runs after checking infrastructure
outcomes, expected coverage and the candidate selection.
To rebuild the appropriate summary from a run's manifest and phase results:
```python
from pathlib import Path
from reme_home.evaluation.artifacts import rebuild_summary
summary = rebuild_summary(Path("/absolute/path/to/run"))
```
New results omit the former `schema_version`, `status`, `execution_complete`,
`execution_mode`, `expected_case_count`, `expected_scored_query_count`,
`scoring_required`, `scoring_complete`, `valid_for_comparison`, `scored_count`,
`unscored_count`, and top-level `query_error_count` from summaries. Use manifest
identity/status/mode and the retained counts instead. The Job returns the run
summary without `cases`, plus `result_dir`, `commit_sha`, `execution_mode` and
execution `status`. Existing result directories are not rewritten automatically.
## Investigating a result
1. Read the manifest to confirm candidate commit, selected cases/queries, judge,
and whether construction was built or reused.
2. Read the summary, then inspect `queries/result.json` for an affected case.
Compare the actual answer with the reference and judge response.
3. Use `answer.log` to trace retrieval and answer generation. Raw
`queries/answers.json` remains available if host judging failed.
4. Inspect `memory_construction/result.json` and `build.log` for construction
failures or unexpected Job behavior.
5. Inspect the extracted pre-query workspace. Its configured memory/session
directories and Git history show memory changes at construction checkpoints.
Use read-only Git commands such as `git log`, `git show`, and `git diff` in
that exported repository. Directory names depend on the runtime configuration.
1. Read the manifest for candidate, selection, mode, judge and construction source.
2. Read the summary, then an affected case's `queries/result.json`. Compare the
answer, reference and judge assessment. Frozen answers survive judge failures.
3. Read `answer.log` for retrieval/answer execution; the host judge is outside it.
4. Read construction `result.json` and `build.log` for Job failures and costs.
5. Inspect the pre-query workspace and its Git history with read-only `git log`,
`git show` and `git diff` to trace session memory checkpoints. Memory/session
directory names depend on runtime configuration.
Memory construction and query answering use different snapshots. Use the
pre-query memory tar to investigate construction or choose a reuse donor; the
post-query archive may contain changes made while answering.
## Scores and comparisons
Candidate failures count as zero in the expected-query denominator.
Infrastructure, artifact, or judge errors make the comparable mean unavailable;
`observed_mean_query_score` reports the available scored observations separately.
Failure artifacts retain structured exception types, HTTP status codes and cause
chains from `Response.metadata["reme.job_error"]`. Infrastructure failures in any
construction job or exception cause invalidate the comparison. Failed jobs without
structured exception details are also treated as infrastructure errors; their
text alone cannot establish a candidate failure. The construction failure-rate
tolerance applies only to classified candidate errors.
Check `valid_for_comparison`, failure details, and coverage before comparing runs.
That flag alone does not establish a complete initial baseline: also verify the
initial candidate commit and full training case/query coverage.
Debug runs (`debug=true`) test execution without host judging and cannot measure
answer quality. Run and case mean scores are null. `expected_query_count` counts
queries expected to execute, while `expected_scored_query_count` is zero; no
queries are awaiting a score. Debug results are always invalid for comparison,
including partial or failed runs.
Construction reuse does not reuse answers or scores: the current candidate
answers again and the host judge scores those new answers.
Failure records preserve structured exception types, HTTP status codes and cause
chains from `Response.metadata["reme.job_error"]`. Network/provider failures in
any construction Job or exception cause invalidate comparison. Failed Jobs without
structured exception details are treated as infrastructure errors. Construction
failure-rate tolerance applies only to classified candidate errors.
Construction reuse answers and judges again with the current candidate; it does
not reuse donor answers or scores.

View file

@ -95,7 +95,7 @@ class FakeWorker:
files = {
"reme_workspace/daily/note.md": "after-answer",
"manifest.json": "{}",
"queries/answers.json": '{"queries": [{"query_id": "fact:1", "answer": "forged"}]}',
"queries/result.json": '{"queries": [{"query_id": "fact:1", "answer": "forged"}]}',
}
for query in self.queries:
files[f"queries/{query.query_id}/answer.log"] = "answer log"
@ -175,17 +175,22 @@ async def test_all_cases_and_artifacts_via_real_job(app, prepared):
response = await app.run_job("home_evaluate", concurrency=1)
assert response.success, response.answer
result = response.answer
assert result["expected_case_count"] == result["scored_count"] == 2
assert result["case_count"] == result["scored_query_count"] == 2
assert result["mean_query_score"] == result["observed_mean_query_score"] == 0.5
run = Path(result["result_dir"])
manifest = json.loads((run / "manifest.json").read_text())
assert manifest["case_ids"] == ["2", "1"]
assert {"case_ids", "query_ids", "reuse_construction_from"}.isdisjoint(manifest["options"])
assert "judge_model" not in manifest
for case_id in ["1", "2"]:
folder = run / "cases" / case_id
assert (folder / "memory_construction/reme_workspace/daily/note.md").read_text() == case_id
assert (folder / "memory_construction/build.log").is_file()
assert (folder / "queries/fact:1/answer.log").is_file()
assert (folder / "evaluation.tar.gz").is_file()
assert not (folder / "evaluation.tar.gz").exists()
assert not (folder / "execution.json").exists()
assert not (folder / "artifacts.json").exists()
assert not (folder / "queries/answers.json").exists()
assert not (folder / "failure.json").exists()
row = json.loads((folder / "queries/result.json").read_text())["queries"][0]
assert row["golden_answer"] == "hello" and row["score"] == 0.5
@ -278,11 +283,19 @@ async def test_build_failure_retains_diagnostics_and_other_cases(app, prepared,
response = await app.run_job("home_evaluate")
assert response.success, response.answer
run = Path(response.answer["result_dir"])
assert response.answer["scored_count"] == 1
assert response.answer["scored_query_count"] == 1
assert response.answer["mean_query_score"] == 0.25
assert (run / "cases/1/full.tar.gz").is_file()
assert (run / "cases/1/memory_construction/build.log").is_file()
assert (run / "cases/1/failure.json").is_file()
summary = json.loads((run / "summary.json").read_text())
assert summary["cases"]["1"]["queries"] == {}
assert summary["query_status_counts"] == {
"completed": 1,
"candidate_failure": 0,
"infra_error": 0,
"unscored": 1,
}
@pytest.mark.asyncio
@ -295,8 +308,8 @@ async def test_judge_failure_preserves_answers_and_fail_fast_result(app, prepare
response = await app.run_job("home_evaluate", case_ids=["1"])
assert not response.success
run = Path(response.answer["result_dir"])
assert not response.answer["scoring_complete"]
assert json.loads((run / "cases/1/queries/answers.json").read_text())["queries"][0]["answer"] == "hello"
assert response.answer["query_status_counts"]["unscored"] == 1
assert json.loads((run / "cases/1/queries/result.json").read_text())["queries"][0]["answer"] == "hello"
assert (run / "cases/1/queries/fact:1/failure.json").is_file()
failed = await app.run_job("home_evaluate", fail_fast=True)
assert not failed.success
@ -317,6 +330,8 @@ async def test_cancellation_closes_pool_and_retains_partial_run(app, prepared, m
await task
assert all(worker.closed for worker in factory.workers)
assert list((prepared / "evaluations").glob("*/*/*/failure.json"))
manifest = json.loads(next((prepared / "evaluations").glob("*/*/*/manifest.json")).read_text())
assert manifest["status"] == "cancelled"
@pytest.mark.asyncio
@ -324,17 +339,17 @@ async def test_debug_skips_judge_and_invalid_export_keeps_answer(app, prepared,
response = await app.run_job("home_evaluate", debug=True)
assert response.success, response.answer
assert app.context.components["agent_wrapper"]["judge"].reply.await_count == 0
assert not response.answer["scoring_required"]
assert response.answer["scored_query_count"] == 0
monkeypatch.setattr(FakeFactory, "bad_export", True)
response = await app.run_job("home_evaluate", case_ids=["1"])
assert not response.success
run = Path(response.answer["result_dir"])
assert (run / "cases/1/queries/answers.json").is_file()
assert response.answer["scored_count"] == 1
assert (run / "cases/1/queries/result.json").is_file()
assert response.answer["scored_query_count"] == 1
assert not list(prepared.rglob("escaped"))
assert "artifact" in (run / "cases/1/failure.json").read_text()
assert response.answer["mean_query_score"] is None
assert not response.answer["valid_for_comparison"]
assert response.answer["case_status_counts"]["infra_error"] == 1
@pytest.mark.asyncio
@ -358,13 +373,11 @@ async def test_mode_controls_data_scoring_and_fail_fast_defaults(app, prepared,
assert manifest["mode"] == ("debug" if debug else "scored")
assert set(summary["cases"]) == ({"debug-1", "debug-2"} if debug else {"1", "2"})
assert summary["expected_query_count"] == 2
assert summary["expected_scored_query_count"] == (0 if debug else 2)
assert summary["scoring_required"] is not debug
assert summary["scoring_complete"] is not debug
assert summary["valid_for_comparison"] is not debug
assert summary["scored_query_count"] == (0 if debug else 2)
assert manifest["status"] == "completed"
assert summary["mean_query_score"] == (None if debug else 0.5)
assert summary["observed_mean_query_score"] == (None if debug else 0.5)
assert summary["unscored_count"] == 0
assert summary["query_status_counts"]["unscored"] == 0
for case in summary["cases"].values():
assert case["mean_score"] == (None if debug else 0.5)
assert app.context.components["agent_wrapper"]["judge"].reply.await_count == (0 if debug else 2)
@ -390,11 +403,11 @@ async def test_debug_empty_answers_fail_without_scoring(app, prepared, monkeypat
summary = json.loads((run / ("partial_summary.json" if fail_fast else "summary.json")).read_text())
assert summary["mean_query_score"] is None
assert summary["observed_mean_query_score"] is None
assert summary["expected_scored_query_count"] == 0
assert not summary["valid_for_comparison"]
assert summary["scored_query_count"] == 0
assert summary["query_status_counts"]["unscored"] == 0
assert summary["case_status_counts"]["candidate_failure"] == 1
assert summary["cases"]["debug-1"]["mean_score"] is None
assert (run / "cases/debug-1/queries/answers.json").is_file()
assert (run / "cases/debug-1/queries/result.json").is_file()
assert app.context.components["agent_wrapper"]["judge"].reply.await_count == 0
@ -503,7 +516,8 @@ async def test_lme_evaluation_calls_host_judge_job(app, prepared, monkeypatch):
assert wrapper.reply.await_count == 2
run = Path(response.answer["result_dir"])
result = json.loads((run / "cases/q1/queries/result.json").read_text())["queries"][0]
assert result["judge_result"]["metadata"]["answer_judgement"] == "yes"
assert result["judge_result"]["metadata"]["raw_answer_judgement"] == "yes"
assert "answer_judgement" not in result["judge_result"]["metadata"]
assert result["score"] == 1
manifest = json.loads((run / "manifest.json").read_text())
assert manifest["judge"]["plugin"] == "lme-judge"
@ -521,7 +535,7 @@ async def test_invalid_lme_judge_is_infra_error_after_one_retry(app, prepared):
assert response.answer["mean_query_score"] is None
assert wrapper.reply.await_count == 2
run = Path(response.answer["result_dir"])
assert (run / "cases/q1/queries/answers.json").is_file()
assert (run / "cases/q1/queries/result.json").is_file()
@pytest.mark.asyncio
@ -680,7 +694,6 @@ async def test_real_job_errors_reach_worker_and_home_summary(app, prepared, monk
code_failure = kind == "code"
assert response.success is code_failure
summary = response.answer
assert summary["valid_for_comparison"] is code_failure
assert summary["case_status_counts"]["infra_error"] == (0 if code_failure else 1)
assert summary["mean_query_score"] == (
0.5 if code_failure and phase == "construction" else 0 if code_failure else None
@ -692,3 +705,21 @@ async def test_real_job_errors_reach_worker_and_home_summary(app, prepared, monk
if kind != "legacy":
assert failure["exception"] == result["exception"]
assert all(worker.closed for worker in FakeFactory.instances[-1].workers)
@pytest.mark.asyncio
async def test_late_cleanup_failure_survives_without_duplicate_execution_record(app, prepared, monkeypatch):
async def close(worker):
worker.closed = True
raise RuntimeError("late cleanup failure")
monkeypatch.setattr(FakeWorker, "close", close)
response = await app.run_job("home_evaluate", case_ids=["1"])
assert not response.success
assert response.answer["mean_query_score"] is None
assert response.answer["observed_mean_query_score"] == 0.5
folder = Path(response.answer["result_dir"]) / "cases/1"
failure = json.loads((folder / "failure.json").read_text())
assert failure["stage"] == "cleanup" and failure["failure_kind"] == "infra_error"
assert "late cleanup failure" in failure["error"]
assert not (folder / "execution.json").exists()

View file

@ -0,0 +1,333 @@
"""Reviewed dev_meta result content, compact statistics and rebuildability."""
# pylint: disable=missing-function-docstring
import asyncio
from copy import deepcopy
import json
import pytest
from reme_home.evaluation.artifacts import average_tokens, read_json, rebuild_summary, summarize, write_json
from reme_home.evaluation.compaction import compact_construction, compact_query
from reme_home.preparation.data.models import CaseSpec, QuerySpec
def test_query_compaction_preserves_evidence_without_duplicate_prompts_or_costs():
raw = {
"question": "What happened?",
"golden_answer": "An event.",
"answer": "The event happened.",
"score": 1.0,
"error": None,
"answer_result": {
"success": True,
"answer": "The event happened.",
"error": None,
"job_call_counts": {"agentic_answer": 2, "search": 3},
"token_usage": {
"answer": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13},
"empty": {"input_tokens": None, "output_tokens": None, "total_tokens": None},
},
"metadata": {
"query": "What happened?",
"response": "The event happened.",
"query_time": "2026-09-12T10:00:00",
"sys_prompt": "reconstructable prompt",
"nested": [{"sys_prompt": "nested prompt", "evidence": "keep"}],
},
},
"judge_result": {
"success": True,
"answer": "yes",
"job_call_counts": {"answer_judge": 1},
"token_usage": {"judge": {"total_tokens": 100}},
"metadata": {
"answer_judgement": "yes",
"raw_answer_judgement": "Yes, the answer is correct.",
"agent_answer": "The event happened.",
"golden_answer": "An event.",
"llm_response": "The event happened.",
"probing_question": "What happened?",
"query": "What happened?",
"question_type": "fact",
"rubric": ["An event"],
},
},
}
original = deepcopy(raw)
result = compact_query(raw)
assert raw == original
assert result["answer_result"] == {
"success": True,
"job_call_counts": {"agentic_answer": 1, "search": 3},
"token_usage": {"answer": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13}},
"metadata": {"query_time": "2026-09-12T10:00:00", "nested": [{"evidence": "keep"}]},
}
assert result["judge_result"] == {
"success": True,
"metadata": {"raw_answer_judgement": "Yes, the answer is correct."},
}
assert result["question"] == raw["question"]
assert result["golden_answer"] == raw["golden_answer"]
assert result["answer"] == raw["answer"]
def test_distinct_failure_details_and_failed_judge_response_are_retained():
raw = {
"answer": "partial answer",
"error": "judge failed",
"score": None,
"answer_result": {"answer": "different", "error": "answer failed"},
"judge_result": {"answer": "invalid judgement", "metadata": {"reason": "diagnostic"}},
"exception": {"type": "TimeoutError", "causes": [{"type": "ReadTimeout"}]},
}
assert compact_query(raw) == raw
def test_construction_compaction_preserves_jobs_and_only_drops_reconstructable_data():
raw = {
"success": True,
"jobs": [
{
"job": "auto_memory",
"memory_checkpoint": {"commit_sha": "abc", "message": "session: s1"},
"result": {
"success": True,
"job_call_counts": {"auto_memory": 2, "daily_list": 1},
"token_usage": {
"answer": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13},
"empty": {"total_tokens": None},
},
"metadata": {
"source_conversation": "[[session/dialog/s1.jsonl]]",
"details": [
{"session_id": "s1", "source_conversation": "[[session/dialog/s1.jsonl]]"},
{"session_id": "s2", "source_conversation": "[[session/dialog/other.jsonl]]"},
],
"plugin_payload": {"index": {"notes": ["keep", "this"]}},
"index": {
"date": "2026-01-01",
"path": "custom/daily/2026-01-01.md",
"created": False,
"changed": True,
"notes": [{"text": "old"}, {"text": "new"}],
},
},
},
},
{
"job": "index_update",
"result": {
"job_call_counts": {"index_update": 1},
"token_usage": {"answer": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}},
},
},
{"job": "index_update", "result": {"token_usage": {"empty": {"total_tokens": None}}}},
],
}
original = deepcopy(raw)
result = compact_construction(raw)
assert raw == original
job = result["jobs"][0]
assert "memory_checkpoint" not in job
assert job["result"]["job_call_counts"] == {"auto_memory": 1, "daily_list": 1}
assert set(job["result"]["token_usage"]) == {"answer"}
metadata = job["result"]["metadata"]
assert metadata["source_conversation"] == "[[session/dialog/s1.jsonl]]"
assert metadata["details"] == [
{"session_id": "s1"},
{"session_id": "s2", "source_conversation": "[[session/dialog/other.jsonl]]"},
]
assert metadata["plugin_payload"] == {"index": {"notes": ["keep", "this"]}}
assert metadata["index"] == {
"date": "2026-01-01",
"path": "custom/daily/2026-01-01.md",
"created": False,
"changed": True,
"notes_count": 2,
}
assert result["jobs"][1]["result"] == {"token_usage": 0}
assert result["jobs"][2]["result"] == {}
def test_summary_matches_reviewed_dev_meta_fields_and_denominators(tmp_path):
cases = [
CaseSpec(
case_id=case_id,
sessions=[],
queries=[QuerySpec(query_id=query_id, question="?", golden_answer="a") for query_id in ids],
)
for case_id, ids in [("one", ["q1", "q2"]), ("two", ["q3"])]
]
write_json(
tmp_path / "cases/one/memory_construction/result.json",
{
"status": "partial_success",
"jobs": [{"result": {"success": True}}, {"result": {"success": False}}],
},
)
write_json(
tmp_path / "cases/one/queries/result.json",
{
"queries": [
{
"query_id": "q1",
"question_type": "fact",
"score": 0.5,
"answer_result": {"token_usage": {"answer": {"total_tokens": 12}}},
},
{
"query_id": "q2",
"question_type": "fact",
"score": None,
"error": "candidate bug",
"answer_result": {},
},
],
},
)
write_json(tmp_path / "cases/two/memory_construction/result.json", {"status": "failure", "jobs": []})
write_json(
tmp_path / "cases/two/failure.json",
{"stage": "construction", "failure_kind": "candidate_failure", "error": "build failed"},
)
summary = summarize(tmp_path, cases)
assert summary == {
"case_count": 2,
"case_status_counts": {"completed": 0, "candidate_failure": 2, "infra_error": 0},
"construction_status_counts": {"success": 0, "partial_success": 1, "failure": 1},
"failure_stage_case_counts": {"construction": 2, "query": 1},
"expected_query_count": 3,
"scored_query_count": 1,
"query_status_counts": {"completed": 1, "candidate_failure": 1, "infra_error": 0, "unscored": 2},
"mean_query_score": 0.5 / 3,
"observed_mean_query_score": 0.5,
"low_score_cases": [{"one": 0.25}],
"question_type_statistics": {
"fact": {"query_count": 2, "average_score": 0.5, "average_query_total_token_usage": {"answer": 6.0}},
},
"cases": {
"one": {
"construction": {"status": "partial_success", "success_job_count": 1, "fail_job_count": 1},
"mean_score": 0.25,
"average_query_total_token_usage": {"answer": 6.0},
"queries": {
"q1": {"question_type": "fact", "score": 0.5, "status": "completed"},
"q2": {"question_type": "fact", "score": None, "status": "candidate_failure"},
},
"construction_error_count": 1,
"query_error_count": 1,
},
"two": {
"construction": {"status": "failure", "success_job_count": 0, "fail_job_count": 0},
"mean_score": 0,
"average_query_total_token_usage": {},
"queries": {},
"stage": "construction",
},
},
}
write_json(
tmp_path / "manifest.json",
{
"status": "completed",
"mode": "scored",
"case_ids": ["one", "two"],
"query_ids": {"one": ["q1", "q2"], "two": ["q3"]},
},
)
assert rebuild_summary(tmp_path) == summary
assert read_json(tmp_path / "summary.json") == summary
failure_path = tmp_path / "cases/two/failure.json"
failure = read_json(failure_path)
failure["failure_kind"] = "infra_error"
write_json(failure_path, failure)
infra = rebuild_summary(tmp_path)
assert infra["mean_query_score"] is None
assert infra["observed_mean_query_score"] == 0.5
assert infra["cases"]["two"]["mean_score"] is None
assert infra["cases"]["one"]["mean_score"] == 0.25
def test_token_averages_exclude_zero_unknown_and_judge_cost_but_count_all_returned_queries():
assert average_tokens(
[
{
"answer_result": {"token_usage": {"answer": {"total_tokens": 12}, "zero": {"total_tokens": 0}}},
"judge_result": {"token_usage": {"judge": {"total_tokens": 100}}},
},
{"answer_result": {"token_usage": {"unknown": {"total_tokens": None}, "bad": {"total_tokens": True}}}},
{},
],
) == {"answer": 4.0}
def test_compact_json_round_trips_strings_and_keeps_small_leaves_inline(tmp_path):
value = {"result": {"token_usage": 0}, "unicode": ["你好", 'quote"\n'], "large": [1, 2, 3]}
path = tmp_path / "result.json"
write_json(path, value, compact_leaf_containers=True)
assert read_json(path) == value
assert '"result": {"token_usage": 0}' in path.read_text()
assert "\\u4f60" not in path.read_text()
with pytest.raises(ValueError):
write_json(path, {"score": float("nan")}, compact_leaf_containers=True)
assert json.loads(path.read_text()) == value
@pytest.mark.asyncio
@pytest.mark.parametrize("stop", [RuntimeError("judge unavailable"), asyncio.CancelledError()])
async def test_judge_interruption_preserves_all_frozen_answers_and_compacts_each_response_once(tmp_path, stop):
from types import SimpleNamespace
from unittest.mock import AsyncMock
from deploy.benchmark_docker.pool import CaseResult
from reme_home.evaluation.component import HomeEvaluation
from reme_home.evaluation.inputs import EvaluationSpec
case = CaseSpec(
case_id="case",
sessions=[],
queries=[QuerySpec(query_id=f"q{i}", question="?", golden_answer="a") for i in range(2)],
)
answers = [
{
"query_id": query.query_id,
"answer": "frozen answer",
"error": None,
"answer_result": {"answer": "frozen answer", "job_call_counts": {"agentic_answer": 2}},
}
for query in case.queries
]
original = deepcopy(answers)
result = CaseResult("case", build={"success": True, "jobs": []}, queries={"queries": answers})
spec = EvaluationSpec(fail_fast=True)
await HomeEvaluation._persist_case(tmp_path, case, result, None, spec) # pylint: disable=protected-access
judge = SimpleNamespace(evaluate=AsyncMock(side_effect=stop))
with pytest.raises(type(stop)):
await HomeEvaluation._score_case( # pylint: disable=protected-access
tmp_path,
case,
"beam",
judge,
asyncio.Semaphore(1),
spec,
answers,
)
rows = read_json(tmp_path / "cases/case/queries/result.json")["queries"]
assert [row["answer"] for row in rows] == ["frozen answer", "frozen answer"]
assert [row["answer_result"]["job_call_counts"] for row in rows] == [{"agentic_answer": 1}] * 2
assert answers == original
assert not (tmp_path / "cases/case/queries/answers.json").exists()
def test_rebuild_rejects_missing_completed_answers_without_replacing_summary(tmp_path):
write_json(
tmp_path / "manifest.json",
{"status": "completed", "case_ids": ["case"], "query_ids": {"case": ["q"]}},
)
write_json(tmp_path / "cases/case/memory_construction/result.json", {"status": "success", "jobs": []})
write_json(tmp_path / "summary.json", {"preserve": "existing summary"})
with pytest.raises(ValueError, match="incomplete query results"):
rebuild_summary(tmp_path)
assert read_json(tmp_path / "summary.json") == {"preserve": "existing summary"}