From 95f9ea7b6130022e4d9787cec68e7729cbbb6f07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=85=B1=E7=89=9B=E8=82=89?= Date: Wed, 28 Oct 2026 17:57:23 +0800 Subject: [PATCH] refactor(home): align evaluation summaries with dev_meta --- plugins/home/README.md | 29 +- .../src/reme_home/evaluation/artifacts.py | 168 +++++++-- .../src/reme_home/evaluation/compaction.py | 177 ++++++++++ .../src/reme_home/evaluation/component.py | 97 +++-- .../home/src/reme_home/evaluation/inputs.py | 2 +- .../runtime_docs/validation-artifact-guide.md | 153 +++++--- plugins/home/tests/test_evaluation.py | 73 ++-- .../home/tests/test_evaluation_artifacts.py | 333 ++++++++++++++++++ 8 files changed, 877 insertions(+), 155 deletions(-) create mode 100644 plugins/home/src/reme_home/evaluation/compaction.py create mode 100644 plugins/home/tests/test_evaluation_artifacts.py diff --git a/plugins/home/README.md b/plugins/home/README.md index 87e91992..462bd574 100644 --- a/plugins/home/README.md +++ b/plugins/home/README.md @@ -241,24 +241,25 @@ Results use `home/evaluations////`: manifest.json summary.json cases// - execution.json memory_construction/ result.json build.log reme_workspace.tar.gz reme_workspace/ - evaluation.tar.gz queries/ - answers.json result.json /answer.log ``` The memory tar is exported **before** queries, retains session Git checkpoints, -and is independent of the post-query evaluation archive. Replayed runs keep an +and is independent of the query execution state. Replayed runs keep an independent donor tar and inherited build log without charging construction tokens again. Answers are persisted before host judging. Per-query results keep -reference/actual answers, scores, judge output, token usage and failures. Summary +reference/actual answers, scores, compact judge output, reported answer token usage +and failures. `queries/result.json` is the sole authoritative answer record; +reconstructable prompts and repeated payloads are removed using the reviewed +`dev_meta` rules. Construction results similarly compact redundant links, Job +counts, token records and cumulative daily-index snapshots. Summary scores use the expected-query denominator; infrastructure/judge/artifact errors make the comparable mean unavailable while retaining the observed-score mean. The Job returns `success=false` for infrastructure/judge/artifact failures, while @@ -273,8 +274,17 @@ their error text. Successful jobs returning empty answers still count as candida failures. The construction failure-rate tolerance applies only to candidate errors: infrastructure or unclassified failures always reject construction, even when another failed job was a candidate error. -`valid_for_comparison` does not assert that an initial baseline covers the full -dataset: the caller must also verify its code and complete case/query coverage. +The manifest retains run identity, selection, mode and execution status. The +summary retains only outcome counts, expected/scored query counts, score means, +low-score cases, question-type statistics and compact case/query mappings. +Candidate answer token summaries average only total tokens per model; detailed +construction costs remain in construction results. Temporary post-query archives +are discarded after extracting logs; separate execution, checksum and raw-answer +JSON copies are not saved. See the +[artifact guide](src/reme_home/runtime_docs/validation-artifact-guide.md) for the +exact fields, rebuild helper and migration from the previous verbose format. +Before comparing runs, verify completed scored execution, infrastructure outcomes, +candidate code and complete case/query coverage. Run/case/query `failure.json` files preserve diagnostics. Interrupted/fail-fast runs publish `partial_summary.json`, not a completed summary. New executions @@ -292,8 +302,9 @@ returns `success=false` for candidate failures as well as infrastructure failure It cannot reuse construction. Case/query selections refer to the selected mode's dataset; omitted selections run all its cases/queries. Debug results retain execution counts and artifacts, but all mean scores are -null and `valid_for_comparison=false`. `expected_scored_query_count` is zero, -while `expected_query_count` still records the expected execution count. +null. `scored_query_count` and `query_status_counts.unscored` are zero, while +`expected_query_count` still records the expected execution count. Debug/scored +mode is recorded in the manifest. The former `dataset_kind` input is rejected; use `debug` instead. Debug results must not be used to assess answer correctness or candidate quality. No real Docker/model evaluation is required by Home's isolated unit tests. diff --git a/plugins/home/src/reme_home/evaluation/artifacts.py b/plugins/home/src/reme_home/evaluation/artifacts.py index e976f30b..9f504817 100644 --- a/plugins/home/src/reme_home/evaluation/artifacts.py +++ b/plugins/home/src/reme_home/evaluation/artifacts.py @@ -7,6 +7,9 @@ from pathlib import Path, PurePosixPath import shutil import tarfile import tempfile +from uuid import uuid4 + +from ..preparation.data.models import CaseSpec, QuerySpec from ..workspace import _validate_directory @@ -32,13 +35,16 @@ def read_json(path: Path): return json.loads(path.read_text(encoding="utf-8")) -def write_json(path: Path, value) -> None: +def write_json(path: Path, value, *, compact_leaf_containers: bool = False) -> None: """Atomically replace a rebuildable or progressively published result.""" path.parent.mkdir(parents=True, exist_ok=True) descriptor, name = tempfile.mkstemp(prefix=".result-", dir=path.parent) try: with os.fdopen(descriptor, "w", encoding="utf-8") as stream: - json.dump(value, stream, ensure_ascii=False, allow_nan=False, indent=2) + if compact_leaf_containers: + stream.write(_dumps_compact_leaf_containers(value)) + else: + json.dump(value, stream, ensure_ascii=False, allow_nan=False, indent=2) stream.write("\n") stream.flush() os.fsync(stream.fileno()) @@ -47,6 +53,69 @@ def write_json(path: Path, value) -> None: Path(name).unlink(missing_ok=True) +def _dumps_compact_leaf_containers(value) -> str: + """Pretty-print JSON while keeping small scalar-only containers on one line.""" + + def is_atom(item) -> bool: + return isinstance(item, (str, int, float, bool, type(None))) + + strings: set[str] = set() + + def collect_strings(item) -> None: + if isinstance(item, str): + strings.add(item) + elif isinstance(item, dict): + for key, child in item.items(): + if isinstance(key, str): + strings.add(key) + collect_strings(child) + elif isinstance(item, (list, tuple)): + for child in item: + collect_strings(child) + + collect_strings(value) + marker_prefix = f"\0meta-reme-singleton-{uuid4().hex}-" + while any(item.startswith(marker_prefix) for item in strings): + marker_prefix = f"\0meta-reme-singleton-{uuid4().hex}-" + + replacements: dict[str, str] = {} + + def replace_leaf_containers(item): + if isinstance(item, dict): + if len(item) == 1 and is_atom(next(iter(item.values()))): + marker = f"{marker_prefix}{len(replacements)}" + replacements[marker] = json.dumps( + item, + ensure_ascii=False, + sort_keys=True, + allow_nan=False, + separators=(",", ": "), + ) + return marker + return {key: replace_leaf_containers(child) for key, child in item.items()} + if isinstance(item, (list, tuple)): + if 1 <= len(item) <= 2 and all(is_atom(child) for child in item): + marker = f"{marker_prefix}{len(replacements)}" + replacements[marker] = json.dumps( + item, + ensure_ascii=False, + sort_keys=True, + allow_nan=False, + separators=(",", ": "), + ) + return marker + children = [replace_leaf_containers(child) for child in item] + return children if isinstance(item, list) else tuple(children) + return item + + transformed = replace_leaf_containers(value) + content = json.dumps(transformed, ensure_ascii=False, indent=2, sort_keys=True, allow_nan=False) + for marker, replacement in replacements.items(): + encoded_marker = json.dumps(marker, ensure_ascii=False) + content = content.replace(encoded_marker, replacement) + return content + + def file_sha256(path: Path) -> str: """Hash potentially large snapshots without loading them into memory.""" with path.open("rb") as stream: @@ -98,14 +167,19 @@ def average_tokens(queries: list[dict]) -> dict: """Average candidate answer total tokens over returned queries, like Meta-ReMe.""" totals = {} for query in queries: - for model, usage in (query.get("answer_result") or {}).get("token_usage", {}).items(): + usage_by_model = (query.get("answer_result") or {}).get("token_usage", {}) + if not isinstance(usage_by_model, dict): + continue + for model, usage in usage_by_model.items(): + if not isinstance(usage, dict): + continue total = usage.get("total_tokens") - if isinstance(total, int) and not isinstance(total, bool): + if isinstance(total, int) and not isinstance(total, bool) and total != 0: totals[model] = totals.get(model, 0) + total return {model: round(total / len(queries), 3) for model, total in totals.items()} if queries else {} -def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = True) -> dict: +def summarize(run: Path, cases: list, *, scoring_required: bool = True) -> dict: """Rebuild compact optimizer evidence with Meta-ReMe's expected-query denominator.""" rows, all_scores, query_results = {}, [], [] case_counts = {"completed": 0, "candidate_failure": 0, "infra_error": 0} @@ -113,18 +187,19 @@ def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = T query_counts = {"completed": 0, "candidate_failure": 0, "infra_error": 0} stages = {"construction": 0, "query": 0} expected = sum(len(case.queries) for case in cases) - expected_scored = expected if scoring_required else 0 for case in cases: - folder = run / "cases" / case.case_id - construction_path = folder / "memory_construction/result.json" + folder = contained(run, f"cases/{case.case_id}", directory=True) + construction_path = contained(folder, "memory_construction/result.json") construction = read_json(construction_path) if construction_path.is_file() else {} - query_path = folder / "queries/result.json" + query_path = contained(folder, "queries/result.json") queries = read_json(query_path).get("queries", []) if query_path.is_file() else [] failure = read_json(folder / "failure.json") if (folder / "failure.json").is_file() else {} + if not construction and not queries and not failure: + continue scores = [query["score"] for query in queries if query.get("score") is not None] all_scores.extend(scores) query_results.extend(queries) - failed = sum(bool(query.get("error")) for query in queries) + failed = sum(query.get("error") is not None for query in queries) has_infra = failure.get("failure_kind") == "infra_error" or any(q.get("infrastructure_error") for q in queries) case_status = ( "infra_error" @@ -136,14 +211,16 @@ def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = T if construction_status == "reused": construction_status = "success" construction_counts[construction_status] += 1 - stages["construction"] += construction_status == "failure" + jobs = [job for job in construction.get("jobs", []) if isinstance(job, dict)] + job_results = [job["result"] if isinstance(job.get("result"), dict) else {} for job in jobs] + construction_errors = sum(not result.get("success") for result in job_results) + stages["construction"] += construction_status == "failure" or bool(construction_errors) stages["query"] += bool(failed) - jobs = construction.get("jobs", []) compact_queries = {} for query in queries: query_status = ( "completed" - if not query.get("error") + if query.get("error") is None else "infra_error" if query.get("infrastructure_error") else "candidate_failure" ) query_counts[query_status] += 1 @@ -155,17 +232,19 @@ def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = T rows[case.case_id] = { "construction": { "status": construction_status, - "success_job_count": sum(job.get("result", {}).get("success") is True for job in jobs), - "fail_job_count": sum(job.get("result", {}).get("success") is not True for job in jobs), + "success_job_count": len(jobs) - construction_errors, + "fail_job_count": construction_errors, }, - "expected_query_count": len(case.queries), - "expected_scored_query_count": len(case.queries) if scoring_required else 0, - "scored_count": len(scores), - "query_error_count": failed, "mean_score": None if has_infra or not scoring_required else sum(scores) / len(case.queries), "average_query_total_token_usage": average_tokens(queries), "queries": compact_queries, } + if failure.get("stage") is not None: + rows[case.case_id]["stage"] = failure["stage"] + if construction_errors: + rows[case.case_id]["construction_error_count"] = construction_errors + if failed: + rows[case.case_id]["query_error_count"] = failed groups = {} for query in query_results: groups.setdefault(str(query.get("question_type") or "unknown"), []).append(query) @@ -178,25 +257,18 @@ def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = T "average_query_total_token_usage": average_tokens(queries), } return { - "schema_version": 1, - "status": status, - "case_count": len(cases), - "expected_case_count": len(cases), + "case_count": len(rows), "case_status_counts": case_counts, "construction_status_counts": construction_counts, "failure_stage_case_counts": stages, "expected_query_count": expected, - "expected_scored_query_count": expected_scored, - "scoring_required": scoring_required, - "scored_count": len(all_scores), "scored_query_count": len(all_scores), - "unscored_count": expected_scored - len(all_scores), - "query_status_counts": {**query_counts, "unscored": expected_scored - len(all_scores)}, - "query_error_count": query_counts["candidate_failure"] + query_counts["infra_error"], - "scoring_complete": scoring_required and len(all_scores) == expected_scored, - "valid_for_comparison": scoring_required and status == "completed" and not case_counts["infra_error"], + "query_status_counts": { + **query_counts, + "unscored": max(expected - len(all_scores), 0) if scoring_required else 0, + }, "mean_query_score": ( - sum(all_scores) / expected_scored if expected_scored and not case_counts["infra_error"] else None + sum(all_scores) / expected if scoring_required and expected and not case_counts["infra_error"] else None ), "observed_mean_query_score": sum(all_scores) / len(all_scores) if all_scores else None, "low_score_cases": [ @@ -213,6 +285,38 @@ def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = T } +def rebuild_summary(run: Path) -> dict: + """Rebuild compact evidence from the manifest and authoritative phase results.""" + manifest = read_json(run / "manifest.json") + cases = [ + CaseSpec( + case_id=case_id, + sessions=[], + queries=[QuerySpec(query_id=query_id, question="", golden_answer=None) for query_id in query_ids], + ) + for case_id, query_ids in manifest["query_ids"].items() + ] + by_id = {case.case_id: case for case in cases} + if manifest.get("status") == "completed": + for case_id in manifest["case_ids"]: + folder = contained(run, f"cases/{case_id}", directory=True) + read_json(contained(folder, "memory_construction/result.json")) + query_path = contained(folder, "queries/result.json") + queries = read_json(query_path)["queries"] if query_path.is_file() else [] + actual_ids = [query["query_id"] for query in queries] + if actual_ids != manifest["query_ids"][case_id]: + if queries or not (folder / "failure.json").is_file(): + raise ValueError(f"Completed evaluation has incomplete query results: {case_id}") + summary = summarize( + run, + [by_id[case_id] for case_id in manifest["case_ids"]], + scoring_required=manifest.get("mode") != "debug", + ) + name = "summary.json" if manifest.get("status") == "completed" else "partial_summary.json" + write_json(run / name, summary, compact_leaf_containers=True) + return summary + + INFRA_TYPES = { "APIConnectionError", "APITimeoutError", diff --git a/plugins/home/src/reme_home/evaluation/compaction.py b/plugins/home/src/reme_home/evaluation/compaction.py new file mode 100644 index 00000000..7db9161b --- /dev/null +++ b/plugins/home/src/reme_home/evaluation/compaction.py @@ -0,0 +1,177 @@ +"""Result compaction rules preserved from dev_meta b53048e4. + +Keep diagnostic evidence while removing reconstructable prompts and duplicate data. +Compaction copies inputs so scoring and failure classification retain raw responses. +""" + +from collections.abc import Mapping +from copy import deepcopy +from pathlib import PurePosixPath +from typing import Any + + +def compact_query(result: dict) -> dict: + """Copy and compact one authoritative query result.""" + result = deepcopy(result) + _compact_query_result(result) + return result + + +def compact_construction(build: dict) -> dict: + """Copy and compact ordered construction jobs using the reviewed rules.""" + persisted_build = deepcopy(build) + _compact_construction_source_conversations(persisted_build) + _compact_construction_index_snapshots(persisted_build) + for job in persisted_build.get("jobs", []): + if not isinstance(job, Mapping): + continue + if isinstance(job, dict): + job.pop("memory_checkpoint", None) + result = job.get("result") + if not isinstance(result, dict): + continue + _compact_outer_job_call_count(job.get("job"), result) + token_usage = result.get("token_usage") + if not isinstance(token_usage, dict): + continue + for backend, usage in list(token_usage.items()): + if isinstance(usage, Mapping) and all( + usage.get(field) is None for field in ("input_tokens", "output_tokens", "total_tokens") + ): + del token_usage[backend] + if not token_usage: + del result["token_usage"] + elif all( + isinstance(usage, Mapping) + and all(usage.get(field) == 0 for field in ("input_tokens", "output_tokens", "total_tokens")) + for usage in token_usage.values() + ): + result["token_usage"] = 0 + return persisted_build + + +def _compact_query_result(result: dict[str, Any]) -> None: + """Remove reconstructable prompts and uninformative token records before persistence.""" + + answer_result = result.get("answer_result") + if isinstance(answer_result, dict): + if answer_result.get("answer") == result.get("answer"): + answer_result.pop("answer", None) + if "error" in result and answer_result.get("error") == result.get("error"): + answer_result.pop("error", None) + _compact_outer_job_call_count("agentic_answer", answer_result) + metadata = answer_result.get("metadata") + if isinstance(metadata, dict): + if metadata.get("query") == result.get("question"): + metadata.pop("query", None) + if metadata.get("response") == result.get("answer"): + metadata.pop("response", None) + + judge_result = result.get("judge_result") + if isinstance(judge_result, dict): + judge_result.pop("token_usage", None) + if result.get("score") is not None: + judge_result.pop("answer", None) + _compact_outer_job_call_count("answer_judge", judge_result) + metadata = judge_result.get("metadata") + if isinstance(metadata, dict): + if metadata.get("raw_answer_judgement") is not None: + metadata.pop("answer_judgement", None) + for field in ( + "agent_answer", + "golden_answer", + "llm_response", + "probing_question", + "query", + "question_type", + "rubric", + ): + metadata.pop(field, None) + + def compact(value: Any) -> None: + if isinstance(value, dict): + value.pop("sys_prompt", None) + token_usage = value.get("token_usage") + if isinstance(token_usage, dict): + for backend, usage in list(token_usage.items()): + if isinstance(usage, Mapping) and all( + usage.get(field) is None for field in ("input_tokens", "output_tokens", "total_tokens") + ): + del token_usage[backend] + if not token_usage: + del value["token_usage"] + for child in value.values(): + compact(child) + elif isinstance(value, list): + for child in value: + compact(child) + + compact(result) + + +def _compact_outer_job_call_count(job_name: Any, result: dict[str, Any]) -> None: + """Remove one call already represented by the enclosing result record.""" + + counts = result.get("job_call_counts") + if not isinstance(job_name, str) or not isinstance(counts, dict): + return + own_count = counts.get(job_name) + if isinstance(own_count, int) and not isinstance(own_count, bool) and own_count > 0: + remaining = own_count - 1 + if remaining: + counts[job_name] = remaining + else: + del counts[job_name] + if not counts: + del result["job_call_counts"] + + +def _compact_construction_index_snapshots(value: Any) -> None: + """Replace cumulative daily-index note snapshots with their counts.""" + + if isinstance(value, dict): + index = value.get("index") + if _is_daily_index_snapshot(index): + index["notes_count"] = len(index.pop("notes")) + for child in value.values(): + _compact_construction_index_snapshots(child) + elif isinstance(value, list): + for child in value: + _compact_construction_index_snapshots(child) + + +def _is_daily_index_snapshot(value: Any) -> bool: + """Recognize the stable ``refresh_day_index`` payload without assuming a configured directory name.""" + + if not isinstance(value, dict) or not isinstance(value.get("notes"), list): + return False + date = value.get("date") + path = value.get("path") + if not isinstance(date, str) or not isinstance(path, str): + return False + index_path = PurePosixPath(path) + return ( + index_path.suffix == ".md" + and index_path.stem == date + and isinstance(value.get("created"), bool) + and isinstance(value.get("changed"), bool) + ) + + +def _compact_construction_source_conversations(value: Any) -> None: + """Drop a transcript link when its sibling session ID already identifies the same file.""" + + if isinstance(value, dict): + session_id = value.get("session_id") + source_conversation = value.get("source_conversation") + if isinstance(session_id, str) and isinstance(source_conversation, str): + source_path = source_conversation.strip() + if source_path.startswith("[[") and source_path[-2:] == "]]": + source_path = source_path[2:-2] + if PurePosixPath(source_path).name == f"{session_id}.jsonl": + del value["source_conversation"] + for child in value.values(): + _compact_construction_source_conversations(child) + elif isinstance(value, list): + for child in value: + _compact_construction_source_conversations(child) diff --git a/plugins/home/src/reme_home/evaluation/component.py b/plugins/home/src/reme_home/evaluation/component.py index dce0ce72..01cfc0ed 100644 --- a/plugins/home/src/reme_home/evaluation/component.py +++ b/plugins/home/src/reme_home/evaluation/component.py @@ -1,7 +1,6 @@ """Home evaluation orchestration using the shared Docker pool and a host judge.""" import asyncio -from dataclasses import asdict import os import shutil import traceback @@ -24,6 +23,7 @@ from .artifacts import ( failure_kind, ) from .inputs import EvaluationSpec, load_inputs, reuse_inputs, snapshot_code +from .compaction import compact_construction, compact_query from ..judge import HomeJudge from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory @@ -146,9 +146,9 @@ class HomeEvaluation(BaseComponent): "execution_mode": "query_only_replay" if source else "full_validation", "construction_source": source, "fingerprints": fingerprints, - "options": spec.model_dump(mode="json"), + "options": spec.model_dump(mode="json", exclude={"case_ids", "query_ids", "reuse_construction_from"}), "judge": self.judge.identity(dataset.dataset), - "judge_model": self.judge.identity(dataset.dataset)["model"], + "status": "running", "started_at": utc_now().isoformat(), } run = self._allocate(branch, commit, manifest) @@ -214,7 +214,15 @@ class HomeEvaluation(BaseComponent): run / "cases" / plan.case_id / "failure.json", {"stage": "artifacts", "error": result.error, "failure_kind": "infra_error"}, ) - await self._score_case(run, by_id[plan.case_id], dataset.dataset, judge, semaphore, spec) + await self._score_case( + run, + by_id[plan.case_id], + dataset.dataset, + judge, + semaphore, + spec, + (result.queries or {}).get("queries", []), + ) results = await BenchmarkDockerPool(factory, concurrency=spec.concurrency).run( plans, @@ -224,9 +232,6 @@ class HomeEvaluation(BaseComponent): ) # Pool teardown can attach late cleanup failures after callbacks. for result in results: - execution = asdict(result) - execution["workspace_snapshot"] = str(result.workspace_snapshot) if result.workspace_snapshot else None - write_json(run / "cases" / result.case_id / "execution.json", execution) if result.error: failure = run / "cases" / result.case_id / "failure.json" if not failure.exists(): @@ -236,22 +241,24 @@ class HomeEvaluation(BaseComponent): if previous.get("error") != result.error: previous.update(cleanup_error=result.error, failure_kind="infra_error") write_json(failure, previous) - summary = summarize(run, cases, status="completed", scoring_required=not spec.debug) - summary.update( - execution_complete=True, - execution_mode=manifest["execution_mode"], - ) + summary = summarize(run, cases, scoring_required=not spec.debug) manifest.update(status="completed", finished_at=utc_now().isoformat()) write_json(run / "manifest.json", manifest) - write_json(run / "summary.json", summary) + write_json(run / "summary.json", summary, compact_leaf_containers=True) return { "result_dir": str(run), "commit_sha": commit, "execution_mode": manifest["execution_mode"], + "status": "completed", **{key: value for key, value in summary.items() if key != "cases"}, } except BaseException as error: if run is not None: + manifest.update( + status="cancelled" if isinstance(error, asyncio.CancelledError) else "failed", + finished_at=utc_now().isoformat(), + ) + write_json(run / "manifest.json", manifest) write_json( run / "failure.json", { @@ -264,11 +271,9 @@ class HomeEvaluation(BaseComponent): partial = summarize( run, cases, - status="cancelled" if isinstance(error, asyncio.CancelledError) else "failed", scoring_required=not spec.debug, ) - partial["execution_complete"] = False - write_json(run / "partial_summary.json", partial) + write_json(run / "partial_summary.json", partial, compact_leaf_containers=True) if isinstance(error, asyncio.CancelledError) or run is None: raise raise RuntimeError(f"Evaluation failed; artifacts: {run}: {error}") from error @@ -277,10 +282,25 @@ class HomeEvaluation(BaseComponent): async def _persist_case(run, case, result, source, spec): folder = run / "cases" / case.case_id memory = folder / "memory_construction" - execution = asdict(result) - execution["workspace_snapshot"] = str(result.workspace_snapshot) if result.workspace_snapshot else None - write_json(folder / "execution.json", execution) - write_json(folder / "queries/answers.json", result.queries or {"queries": []}) + # Freeze every returned answer in the sole authoritative query file + # before extraction or judging can fail. Unjudged answers stay present. + by_id = {row.get("query_id"): row for row in (result.queries or {}).get("queries", [])} + rows = [ + compact_query( + { + **by_id[query.query_id], + "query_id": query.query_id, + "question": query.question, + "golden_answer": query.golden_answer, + "question_type": query.metadata.get("question_type"), + "score": None, + "judge_result": None, + }, + ) + for query in case.queries + if query.query_id in by_id + ] + write_json(folder / "queries/result.json", {"queries": rows}, compact_leaf_containers=True) if result.error: stage = result.failure_stage or "execution" exception = result.exception @@ -309,7 +329,7 @@ class HomeEvaluation(BaseComponent): archive = memory / "reme_workspace.tar.gz" if archive.is_file(): construction["workspace_sha256"] = await blocking(file_sha256, archive) - write_json(memory / "result.json", construction) + write_json(memory / "result.json", compact_construction(construction), compact_leaf_containers=True) archive = memory / "reme_workspace.tar.gz" if archive.is_file(): await blocking(inspect_archive, archive, spec.max_artifact_bytes, memory / "reme_workspace") @@ -325,25 +345,21 @@ class HomeEvaluation(BaseComponent): target = folder / "queries" / query.query_id / "answer.log" target.parent.mkdir(parents=True, exist_ok=True) await blocking(shutil.copyfile, log, target) - # Do not merge the container's queries tree: it could overwrite - # host-frozen answers.json or host-only judge results. The complete - # untrusted tree remains available in evaluation.tar.gz for inspection. + # Container results cannot overwrite host-frozen answers or scores. await blocking(shutil.rmtree, extracted) - integrity = {} - for relative in ("memory_construction/reme_workspace.tar.gz", "evaluation.tar.gz", "full.tar.gz"): - artifact = folder / relative - if artifact.is_file(): - integrity[relative] = await blocking(file_sha256, artifact) - write_json(folder / "artifacts.json", {"sha256": integrity}) + evaluation.unlink() @staticmethod - async def _score_case(run, case, dataset, judge, semaphore, spec): + async def _score_case(run, case, dataset, judge, semaphore, spec, answers): folder = run / "cases" / case.case_id - answers = read_json(folder / "queries/answers.json").get("queries", []) by_id = {row.get("query_id"): row for row in answers} - rows = [] + rows = read_json(folder / "queries/result.json").get("queries", []) + positions = {row["query_id"]: index for index, row in enumerate(rows)} + case_failure = read_json(folder / "failure.json") if (folder / "failure.json").exists() else {} for query in case.queries: raw = by_id.get(query.query_id, {}) + if not raw and case_failure: + continue # Construction failures leave queries unassessed. row = { **raw, "query_id": query.query_id, @@ -353,10 +369,9 @@ class HomeEvaluation(BaseComponent): "score": None, "judge_result": None, } - if not isinstance(row.get("answer"), str) or not row["answer"].strip() or row.get("error"): + if not isinstance(row.get("answer"), str) or not row["answer"].strip() or row.get("error") is not None: row["error"] = row.get("error") or "No successful answer (construction or execution failed)" row["failure_stage"] = "answer" - case_failure = read_json(folder / "failure.json") if (folder / "failure.json").exists() else {} row["failure_kind"] = ( failure_kind(row["exception"], "answer") if row.get("exception") @@ -376,10 +391,14 @@ class HomeEvaluation(BaseComponent): failure_kind="infra_error", infrastructure_error=True, ) - rows.append(row) + compact = compact_query(row) + if query.query_id in positions: + rows[positions[query.query_id]] = compact + else: + rows.append(compact) # Publish each score; a later judge failure cannot erase frozen answers. - write_json(folder / "queries/result.json", {"queries": rows}) - if row.get("error"): - write_json(folder / "queries" / query.query_id / "failure.json", row) + write_json(folder / "queries/result.json", {"queries": rows}, compact_leaf_containers=True) + if row.get("error") is not None: + write_json(folder / "queries" / query.query_id / "failure.json", compact, compact_leaf_containers=True) if spec.fail_fast: raise RuntimeError(f"Query {case.case_id}/{query.query_id} failed: {row['error']}") diff --git a/plugins/home/src/reme_home/evaluation/inputs.py b/plugins/home/src/reme_home/evaluation/inputs.py index 8f7d164c..23d7a4c1 100644 --- a/plugins/home/src/reme_home/evaluation/inputs.py +++ b/plugins/home/src/reme_home/evaluation/inputs.py @@ -213,7 +213,7 @@ def reuse_inputs(home: Path, spec: EvaluationSpec, cases: list, fingerprints: di raise ValueError("reuse_construction_from must be inside home/evaluations") manifest = read_json(source / "manifest.json") summary = read_json(source / "summary.json") - if manifest.get("dataset_kind") != "training-set" or summary.get("status") != "completed": + if manifest.get("dataset_kind") != "training-set" or manifest.get("status", summary.get("status")) != "completed": raise ValueError("Reuse requires a completed training-set evaluation") for key in ("dataset", "construction_config"): if manifest.get("fingerprints", {}).get(key) != fingerprints[key]: diff --git a/plugins/home/src/reme_home/runtime_docs/validation-artifact-guide.md b/plugins/home/src/reme_home/runtime_docs/validation-artifact-guide.md index 4dfe831b..b6cd3f06 100644 --- a/plugins/home/src/reme_home/runtime_docs/validation-artifact-guide.md +++ b/plugins/home/src/reme_home/runtime_docs/validation-artifact-guide.md @@ -4,6 +4,7 @@ `home/evaluations////` inside the outer ReMe workspace. Commit prefixes start at seven characters and extend on collision. Run IDs increase within that commit directory; prior results are preserved. +Result compaction and summary fields follow the reviewed `dev_meta` format. ## Saved files @@ -11,65 +12,111 @@ Paths in this table are relative to one run directory: | Path | Meaning | | --- | --- | -| `manifest.json` | Candidate commit, case/query selection, execution mode, input/configuration fingerprints, host judge identity, and run status. | -| `summary.json` | Completed run summary: aggregate scores, expected/scored counts, case/query outcomes, low-score cases, and comparison validity. | -| `partial_summary.json` | Partial results after an interrupted or failed run; not a completed baseline. | +| `manifest.json` | Candidate commit, selected case/query IDs, dataset, execution mode, input/configuration fingerprints, host judge identity, options, timestamps and run status. Retained for reproducibility and rebuilding summaries; identity is not repeated in the summary. | +| `summary.json` | Rebuildable evidence from a completed execution. Its fields are described below. | +| `partial_summary.json` | Available evidence after cancellation or failure; not a completed baseline. | | `failure.json` | Run-level failure details, when present. | -| `cases//execution.json` | Docker pool execution outcome, including execution/cleanup failures. | -| `cases//artifacts.json` | Checksums of exported archives. | -| `cases//memory_construction/result.json` | Construction outcome and ordered Job results, including available errors, metadata, and token usage. | -| `cases//memory_construction/build.log` | Construction log. Replayed runs inherit the donor log. | -| `cases//memory_construction/reme_workspace.tar.gz` | Workspace snapshot exported before answering queries; the donor for construction reuse. | +| `cases//memory_construction/result.json` | Ordered construction Job results with success, errors, metadata, answers, reported token usage and inner Job call counts. Replays record their source and snapshot checksum without charging construction again. | +| `cases//memory_construction/build.log` | Construction log. Replays inherit the donor log. | +| `cases//memory_construction/reme_workspace.tar.gz` | Checksum-verified workspace snapshot exported before answering; the donor for construction reuse. | | `cases//memory_construction/reme_workspace/` | Extracted construction workspace, including memory files and session Git checkpoints when exported. | -| `cases//evaluation.tar.gz` | Exported post-query evaluation artifacts. | -| `cases//queries/answers.json` | Raw answers saved by the host before judging and artifact extraction. | -| `cases//queries/result.json` | Per-query question, reference and actual answer, score, answer response, judge response, and failure information. | +| `cases//queries/result.json` | The sole authoritative query record: question, reference and actual answer, question type, score, compact answer/judge responses and failures. All returned answers are frozen here before extraction and host judging; scores are published progressively. | | `cases//queries//answer.log` | Query execution log, when available. | | Case/query `failure.json` | Failure stage and diagnostics, when present. | +| `cases//full.tar.gz` | Best-effort sandbox diagnostics for execution failures, when available. | -Exported files depend on how far execution progressed. Infrastructure failures -and timeouts may prevent logs or full workspace archives from being recovered. -Inspect failure records before assuming that an absent artifact means the -candidate produced no memory or answer. +Successful exports retain query logs, then discard the temporary post-query +`evaluation.tar.gz`. There is no separate `execution.json`, `artifacts.json`, +`queries/answers.json` or per-query `result.json`. A failed extraction can leave +its archive for diagnosis. Exported files depend on execution progress; inspect +failure records before interpreting an absent workspace or log. + +Query compaction removes reconstructable `sys_prompt` values, duplicate nested +answers/questions/errors, the enclosing Job's one call, judge token usage, +duplicate parsed judgements when raw judgements exist, and repeated judge inputs. +Distinct errors, raw judgements, rubric assessments and reported answer token +metrics remain. Unreported token backends are omitted. + +Construction compaction removes checkpoint descriptors already available in Git, +one enclosing Job call, unreported token backends, and transcript links identified +by a sibling session ID. Recognized cumulative daily-index `notes` snapshots become +`notes_count`; unrelated plugin payloads remain. All-zero construction token +records become `0`. Detailed construction costs stay in this case artifact. +Small scalar containers are written on one line, following `dev_meta` formatting. + +## Summary fields + +The run summary contains only: + +- `case_count`, `case_status_counts`, `construction_status_counts`, and + `failure_stage_case_counts` (construction/query; tolerated construction failures + still count as construction errors). +- `expected_query_count`, `scored_query_count`, and `query_status_counts` + (completed, candidate failure, infrastructure error, unscored). +- `mean_query_score`, `observed_mean_query_score`, `low_score_cases`, + `question_type_statistics`, and `cases`. + +Each `cases[case_id]` contains construction `status`, `success_job_count` and +`fail_job_count`; `mean_score`; `average_query_total_token_usage`; and a `queries` +map. Each query entry contains only `question_type`, `score` and `status`. +`stage`, `construction_error_count` and `query_error_count` appear only when +applicable. There is no redundant case status or per-case coverage accounting. +Questions, answers, errors, Job details and construction token usage stay in +phase artifacts, outside the summary. + +Token summaries contain only average candidate answer **total** tokens per model, +divided by all returned queries, including failures and queries without reported +usage. Zero and unknown token totals are omitted. Judge/construction costs and +input/output token breakdowns are not aggregated. Question-type statistics contain +query count, average of available scores (rounded to three decimals), and the same +answer token averages across cases. Missing/empty types form `unknown`. + +Candidate failures count as zero in the expected-query score denominator. +Construction failures leave queries unassessed rather than inventing query +failures. Infrastructure, artifact, cleanup or judge errors make the comparable +run mean unavailable; `observed_mean_query_score` averages available scores only. +`low_score_cases` contains `{case_id: mean_score}` for cases with at least one +scored query below `1`, sorted by mean then case ID; unavailable case means are +omitted. + +Run status and debug/scored mode are in the manifest. Debug runs skip judging; +run and case means are null, `scored_query_count` and the unscored count are zero, +while `expected_query_count` records execution coverage. They cannot measure +answer quality. Only compare completed scored runs after checking infrastructure +outcomes, expected coverage and the candidate selection. + +To rebuild the appropriate summary from a run's manifest and phase results: + +```python +from pathlib import Path +from reme_home.evaluation.artifacts import rebuild_summary + +summary = rebuild_summary(Path("/absolute/path/to/run")) +``` + +New results omit the former `schema_version`, `status`, `execution_complete`, +`execution_mode`, `expected_case_count`, `expected_scored_query_count`, +`scoring_required`, `scoring_complete`, `valid_for_comparison`, `scored_count`, +`unscored_count`, and top-level `query_error_count` from summaries. Use manifest +identity/status/mode and the retained counts instead. The Job returns the run +summary without `cases`, plus `result_dir`, `commit_sha`, `execution_mode` and +execution `status`. Existing result directories are not rewritten automatically. ## Investigating a result -1. Read the manifest to confirm candidate commit, selected cases/queries, judge, - and whether construction was built or reused. -2. Read the summary, then inspect `queries/result.json` for an affected case. - Compare the actual answer with the reference and judge response. -3. Use `answer.log` to trace retrieval and answer generation. Raw - `queries/answers.json` remains available if host judging failed. -4. Inspect `memory_construction/result.json` and `build.log` for construction - failures or unexpected Job behavior. -5. Inspect the extracted pre-query workspace. Its configured memory/session - directories and Git history show memory changes at construction checkpoints. - Use read-only Git commands such as `git log`, `git show`, and `git diff` in - that exported repository. Directory names depend on the runtime configuration. +1. Read the manifest for candidate, selection, mode, judge and construction source. +2. Read the summary, then an affected case's `queries/result.json`. Compare the + answer, reference and judge assessment. Frozen answers survive judge failures. +3. Read `answer.log` for retrieval/answer execution; the host judge is outside it. +4. Read construction `result.json` and `build.log` for Job failures and costs. +5. Inspect the pre-query workspace and its Git history with read-only `git log`, + `git show` and `git diff` to trace session memory checkpoints. Memory/session + directory names depend on runtime configuration. -Memory construction and query answering use different snapshots. Use the -pre-query memory tar to investigate construction or choose a reuse donor; the -post-query archive may contain changes made while answering. - -## Scores and comparisons - -Candidate failures count as zero in the expected-query denominator. -Infrastructure, artifact, or judge errors make the comparable mean unavailable; -`observed_mean_query_score` reports the available scored observations separately. -Failure artifacts retain structured exception types, HTTP status codes and cause -chains from `Response.metadata["reme.job_error"]`. Infrastructure failures in any -construction job or exception cause invalidate the comparison. Failed jobs without -structured exception details are also treated as infrastructure errors; their -text alone cannot establish a candidate failure. The construction failure-rate -tolerance applies only to classified candidate errors. -Check `valid_for_comparison`, failure details, and coverage before comparing runs. -That flag alone does not establish a complete initial baseline: also verify the -initial candidate commit and full training case/query coverage. - -Debug runs (`debug=true`) test execution without host judging and cannot measure -answer quality. Run and case mean scores are null. `expected_query_count` counts -queries expected to execute, while `expected_scored_query_count` is zero; no -queries are awaiting a score. Debug results are always invalid for comparison, -including partial or failed runs. -Construction reuse does not reuse answers or scores: the current candidate -answers again and the host judge scores those new answers. +Failure records preserve structured exception types, HTTP status codes and cause +chains from `Response.metadata["reme.job_error"]`. Network/provider failures in +any construction Job or exception cause invalidate comparison. Failed Jobs without +structured exception details are treated as infrastructure errors. Construction +failure-rate tolerance applies only to classified candidate errors. +Construction reuse answers and judges again with the current candidate; it does +not reuse donor answers or scores. diff --git a/plugins/home/tests/test_evaluation.py b/plugins/home/tests/test_evaluation.py index c75ac0ad..5af421a1 100644 --- a/plugins/home/tests/test_evaluation.py +++ b/plugins/home/tests/test_evaluation.py @@ -95,7 +95,7 @@ class FakeWorker: files = { "reme_workspace/daily/note.md": "after-answer", "manifest.json": "{}", - "queries/answers.json": '{"queries": [{"query_id": "fact:1", "answer": "forged"}]}', + "queries/result.json": '{"queries": [{"query_id": "fact:1", "answer": "forged"}]}', } for query in self.queries: files[f"queries/{query.query_id}/answer.log"] = "answer log" @@ -175,17 +175,22 @@ async def test_all_cases_and_artifacts_via_real_job(app, prepared): response = await app.run_job("home_evaluate", concurrency=1) assert response.success, response.answer result = response.answer - assert result["expected_case_count"] == result["scored_count"] == 2 + assert result["case_count"] == result["scored_query_count"] == 2 assert result["mean_query_score"] == result["observed_mean_query_score"] == 0.5 run = Path(result["result_dir"]) manifest = json.loads((run / "manifest.json").read_text()) assert manifest["case_ids"] == ["2", "1"] + assert {"case_ids", "query_ids", "reuse_construction_from"}.isdisjoint(manifest["options"]) + assert "judge_model" not in manifest for case_id in ["1", "2"]: folder = run / "cases" / case_id assert (folder / "memory_construction/reme_workspace/daily/note.md").read_text() == case_id assert (folder / "memory_construction/build.log").is_file() assert (folder / "queries/fact:1/answer.log").is_file() - assert (folder / "evaluation.tar.gz").is_file() + assert not (folder / "evaluation.tar.gz").exists() + assert not (folder / "execution.json").exists() + assert not (folder / "artifacts.json").exists() + assert not (folder / "queries/answers.json").exists() assert not (folder / "failure.json").exists() row = json.loads((folder / "queries/result.json").read_text())["queries"][0] assert row["golden_answer"] == "hello" and row["score"] == 0.5 @@ -278,11 +283,19 @@ async def test_build_failure_retains_diagnostics_and_other_cases(app, prepared, response = await app.run_job("home_evaluate") assert response.success, response.answer run = Path(response.answer["result_dir"]) - assert response.answer["scored_count"] == 1 + assert response.answer["scored_query_count"] == 1 assert response.answer["mean_query_score"] == 0.25 assert (run / "cases/1/full.tar.gz").is_file() assert (run / "cases/1/memory_construction/build.log").is_file() assert (run / "cases/1/failure.json").is_file() + summary = json.loads((run / "summary.json").read_text()) + assert summary["cases"]["1"]["queries"] == {} + assert summary["query_status_counts"] == { + "completed": 1, + "candidate_failure": 0, + "infra_error": 0, + "unscored": 1, + } @pytest.mark.asyncio @@ -295,8 +308,8 @@ async def test_judge_failure_preserves_answers_and_fail_fast_result(app, prepare response = await app.run_job("home_evaluate", case_ids=["1"]) assert not response.success run = Path(response.answer["result_dir"]) - assert not response.answer["scoring_complete"] - assert json.loads((run / "cases/1/queries/answers.json").read_text())["queries"][0]["answer"] == "hello" + assert response.answer["query_status_counts"]["unscored"] == 1 + assert json.loads((run / "cases/1/queries/result.json").read_text())["queries"][0]["answer"] == "hello" assert (run / "cases/1/queries/fact:1/failure.json").is_file() failed = await app.run_job("home_evaluate", fail_fast=True) assert not failed.success @@ -317,6 +330,8 @@ async def test_cancellation_closes_pool_and_retains_partial_run(app, prepared, m await task assert all(worker.closed for worker in factory.workers) assert list((prepared / "evaluations").glob("*/*/*/failure.json")) + manifest = json.loads(next((prepared / "evaluations").glob("*/*/*/manifest.json")).read_text()) + assert manifest["status"] == "cancelled" @pytest.mark.asyncio @@ -324,17 +339,17 @@ async def test_debug_skips_judge_and_invalid_export_keeps_answer(app, prepared, response = await app.run_job("home_evaluate", debug=True) assert response.success, response.answer assert app.context.components["agent_wrapper"]["judge"].reply.await_count == 0 - assert not response.answer["scoring_required"] + assert response.answer["scored_query_count"] == 0 monkeypatch.setattr(FakeFactory, "bad_export", True) response = await app.run_job("home_evaluate", case_ids=["1"]) assert not response.success run = Path(response.answer["result_dir"]) - assert (run / "cases/1/queries/answers.json").is_file() - assert response.answer["scored_count"] == 1 + assert (run / "cases/1/queries/result.json").is_file() + assert response.answer["scored_query_count"] == 1 assert not list(prepared.rglob("escaped")) assert "artifact" in (run / "cases/1/failure.json").read_text() assert response.answer["mean_query_score"] is None - assert not response.answer["valid_for_comparison"] + assert response.answer["case_status_counts"]["infra_error"] == 1 @pytest.mark.asyncio @@ -358,13 +373,11 @@ async def test_mode_controls_data_scoring_and_fail_fast_defaults(app, prepared, assert manifest["mode"] == ("debug" if debug else "scored") assert set(summary["cases"]) == ({"debug-1", "debug-2"} if debug else {"1", "2"}) assert summary["expected_query_count"] == 2 - assert summary["expected_scored_query_count"] == (0 if debug else 2) - assert summary["scoring_required"] is not debug - assert summary["scoring_complete"] is not debug - assert summary["valid_for_comparison"] is not debug + assert summary["scored_query_count"] == (0 if debug else 2) + assert manifest["status"] == "completed" assert summary["mean_query_score"] == (None if debug else 0.5) assert summary["observed_mean_query_score"] == (None if debug else 0.5) - assert summary["unscored_count"] == 0 + assert summary["query_status_counts"]["unscored"] == 0 for case in summary["cases"].values(): assert case["mean_score"] == (None if debug else 0.5) assert app.context.components["agent_wrapper"]["judge"].reply.await_count == (0 if debug else 2) @@ -390,11 +403,11 @@ async def test_debug_empty_answers_fail_without_scoring(app, prepared, monkeypat summary = json.loads((run / ("partial_summary.json" if fail_fast else "summary.json")).read_text()) assert summary["mean_query_score"] is None assert summary["observed_mean_query_score"] is None - assert summary["expected_scored_query_count"] == 0 - assert not summary["valid_for_comparison"] + assert summary["scored_query_count"] == 0 + assert summary["query_status_counts"]["unscored"] == 0 assert summary["case_status_counts"]["candidate_failure"] == 1 assert summary["cases"]["debug-1"]["mean_score"] is None - assert (run / "cases/debug-1/queries/answers.json").is_file() + assert (run / "cases/debug-1/queries/result.json").is_file() assert app.context.components["agent_wrapper"]["judge"].reply.await_count == 0 @@ -503,7 +516,8 @@ async def test_lme_evaluation_calls_host_judge_job(app, prepared, monkeypatch): assert wrapper.reply.await_count == 2 run = Path(response.answer["result_dir"]) result = json.loads((run / "cases/q1/queries/result.json").read_text())["queries"][0] - assert result["judge_result"]["metadata"]["answer_judgement"] == "yes" + assert result["judge_result"]["metadata"]["raw_answer_judgement"] == "yes" + assert "answer_judgement" not in result["judge_result"]["metadata"] assert result["score"] == 1 manifest = json.loads((run / "manifest.json").read_text()) assert manifest["judge"]["plugin"] == "lme-judge" @@ -521,7 +535,7 @@ async def test_invalid_lme_judge_is_infra_error_after_one_retry(app, prepared): assert response.answer["mean_query_score"] is None assert wrapper.reply.await_count == 2 run = Path(response.answer["result_dir"]) - assert (run / "cases/q1/queries/answers.json").is_file() + assert (run / "cases/q1/queries/result.json").is_file() @pytest.mark.asyncio @@ -680,7 +694,6 @@ async def test_real_job_errors_reach_worker_and_home_summary(app, prepared, monk code_failure = kind == "code" assert response.success is code_failure summary = response.answer - assert summary["valid_for_comparison"] is code_failure assert summary["case_status_counts"]["infra_error"] == (0 if code_failure else 1) assert summary["mean_query_score"] == ( 0.5 if code_failure and phase == "construction" else 0 if code_failure else None @@ -692,3 +705,21 @@ async def test_real_job_errors_reach_worker_and_home_summary(app, prepared, monk if kind != "legacy": assert failure["exception"] == result["exception"] assert all(worker.closed for worker in FakeFactory.instances[-1].workers) + + +@pytest.mark.asyncio +async def test_late_cleanup_failure_survives_without_duplicate_execution_record(app, prepared, monkeypatch): + async def close(worker): + worker.closed = True + raise RuntimeError("late cleanup failure") + + monkeypatch.setattr(FakeWorker, "close", close) + response = await app.run_job("home_evaluate", case_ids=["1"]) + assert not response.success + assert response.answer["mean_query_score"] is None + assert response.answer["observed_mean_query_score"] == 0.5 + folder = Path(response.answer["result_dir"]) / "cases/1" + failure = json.loads((folder / "failure.json").read_text()) + assert failure["stage"] == "cleanup" and failure["failure_kind"] == "infra_error" + assert "late cleanup failure" in failure["error"] + assert not (folder / "execution.json").exists() diff --git a/plugins/home/tests/test_evaluation_artifacts.py b/plugins/home/tests/test_evaluation_artifacts.py new file mode 100644 index 00000000..4ea0ea3e --- /dev/null +++ b/plugins/home/tests/test_evaluation_artifacts.py @@ -0,0 +1,333 @@ +"""Reviewed dev_meta result content, compact statistics and rebuildability.""" + +# pylint: disable=missing-function-docstring + +import asyncio +from copy import deepcopy +import json + +import pytest + +from reme_home.evaluation.artifacts import average_tokens, read_json, rebuild_summary, summarize, write_json +from reme_home.evaluation.compaction import compact_construction, compact_query +from reme_home.preparation.data.models import CaseSpec, QuerySpec + + +def test_query_compaction_preserves_evidence_without_duplicate_prompts_or_costs(): + raw = { + "question": "What happened?", + "golden_answer": "An event.", + "answer": "The event happened.", + "score": 1.0, + "error": None, + "answer_result": { + "success": True, + "answer": "The event happened.", + "error": None, + "job_call_counts": {"agentic_answer": 2, "search": 3}, + "token_usage": { + "answer": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13}, + "empty": {"input_tokens": None, "output_tokens": None, "total_tokens": None}, + }, + "metadata": { + "query": "What happened?", + "response": "The event happened.", + "query_time": "2026-09-12T10:00:00", + "sys_prompt": "reconstructable prompt", + "nested": [{"sys_prompt": "nested prompt", "evidence": "keep"}], + }, + }, + "judge_result": { + "success": True, + "answer": "yes", + "job_call_counts": {"answer_judge": 1}, + "token_usage": {"judge": {"total_tokens": 100}}, + "metadata": { + "answer_judgement": "yes", + "raw_answer_judgement": "Yes, the answer is correct.", + "agent_answer": "The event happened.", + "golden_answer": "An event.", + "llm_response": "The event happened.", + "probing_question": "What happened?", + "query": "What happened?", + "question_type": "fact", + "rubric": ["An event"], + }, + }, + } + original = deepcopy(raw) + result = compact_query(raw) + assert raw == original + assert result["answer_result"] == { + "success": True, + "job_call_counts": {"agentic_answer": 1, "search": 3}, + "token_usage": {"answer": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13}}, + "metadata": {"query_time": "2026-09-12T10:00:00", "nested": [{"evidence": "keep"}]}, + } + assert result["judge_result"] == { + "success": True, + "metadata": {"raw_answer_judgement": "Yes, the answer is correct."}, + } + assert result["question"] == raw["question"] + assert result["golden_answer"] == raw["golden_answer"] + assert result["answer"] == raw["answer"] + + +def test_distinct_failure_details_and_failed_judge_response_are_retained(): + raw = { + "answer": "partial answer", + "error": "judge failed", + "score": None, + "answer_result": {"answer": "different", "error": "answer failed"}, + "judge_result": {"answer": "invalid judgement", "metadata": {"reason": "diagnostic"}}, + "exception": {"type": "TimeoutError", "causes": [{"type": "ReadTimeout"}]}, + } + assert compact_query(raw) == raw + + +def test_construction_compaction_preserves_jobs_and_only_drops_reconstructable_data(): + raw = { + "success": True, + "jobs": [ + { + "job": "auto_memory", + "memory_checkpoint": {"commit_sha": "abc", "message": "session: s1"}, + "result": { + "success": True, + "job_call_counts": {"auto_memory": 2, "daily_list": 1}, + "token_usage": { + "answer": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13}, + "empty": {"total_tokens": None}, + }, + "metadata": { + "source_conversation": "[[session/dialog/s1.jsonl]]", + "details": [ + {"session_id": "s1", "source_conversation": "[[session/dialog/s1.jsonl]]"}, + {"session_id": "s2", "source_conversation": "[[session/dialog/other.jsonl]]"}, + ], + "plugin_payload": {"index": {"notes": ["keep", "this"]}}, + "index": { + "date": "2026-01-01", + "path": "custom/daily/2026-01-01.md", + "created": False, + "changed": True, + "notes": [{"text": "old"}, {"text": "new"}], + }, + }, + }, + }, + { + "job": "index_update", + "result": { + "job_call_counts": {"index_update": 1}, + "token_usage": {"answer": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}}, + }, + }, + {"job": "index_update", "result": {"token_usage": {"empty": {"total_tokens": None}}}}, + ], + } + original = deepcopy(raw) + result = compact_construction(raw) + assert raw == original + job = result["jobs"][0] + assert "memory_checkpoint" not in job + assert job["result"]["job_call_counts"] == {"auto_memory": 1, "daily_list": 1} + assert set(job["result"]["token_usage"]) == {"answer"} + metadata = job["result"]["metadata"] + assert metadata["source_conversation"] == "[[session/dialog/s1.jsonl]]" + assert metadata["details"] == [ + {"session_id": "s1"}, + {"session_id": "s2", "source_conversation": "[[session/dialog/other.jsonl]]"}, + ] + assert metadata["plugin_payload"] == {"index": {"notes": ["keep", "this"]}} + assert metadata["index"] == { + "date": "2026-01-01", + "path": "custom/daily/2026-01-01.md", + "created": False, + "changed": True, + "notes_count": 2, + } + assert result["jobs"][1]["result"] == {"token_usage": 0} + assert result["jobs"][2]["result"] == {} + + +def test_summary_matches_reviewed_dev_meta_fields_and_denominators(tmp_path): + cases = [ + CaseSpec( + case_id=case_id, + sessions=[], + queries=[QuerySpec(query_id=query_id, question="?", golden_answer="a") for query_id in ids], + ) + for case_id, ids in [("one", ["q1", "q2"]), ("two", ["q3"])] + ] + write_json( + tmp_path / "cases/one/memory_construction/result.json", + { + "status": "partial_success", + "jobs": [{"result": {"success": True}}, {"result": {"success": False}}], + }, + ) + write_json( + tmp_path / "cases/one/queries/result.json", + { + "queries": [ + { + "query_id": "q1", + "question_type": "fact", + "score": 0.5, + "answer_result": {"token_usage": {"answer": {"total_tokens": 12}}}, + }, + { + "query_id": "q2", + "question_type": "fact", + "score": None, + "error": "candidate bug", + "answer_result": {}, + }, + ], + }, + ) + write_json(tmp_path / "cases/two/memory_construction/result.json", {"status": "failure", "jobs": []}) + write_json( + tmp_path / "cases/two/failure.json", + {"stage": "construction", "failure_kind": "candidate_failure", "error": "build failed"}, + ) + summary = summarize(tmp_path, cases) + assert summary == { + "case_count": 2, + "case_status_counts": {"completed": 0, "candidate_failure": 2, "infra_error": 0}, + "construction_status_counts": {"success": 0, "partial_success": 1, "failure": 1}, + "failure_stage_case_counts": {"construction": 2, "query": 1}, + "expected_query_count": 3, + "scored_query_count": 1, + "query_status_counts": {"completed": 1, "candidate_failure": 1, "infra_error": 0, "unscored": 2}, + "mean_query_score": 0.5 / 3, + "observed_mean_query_score": 0.5, + "low_score_cases": [{"one": 0.25}], + "question_type_statistics": { + "fact": {"query_count": 2, "average_score": 0.5, "average_query_total_token_usage": {"answer": 6.0}}, + }, + "cases": { + "one": { + "construction": {"status": "partial_success", "success_job_count": 1, "fail_job_count": 1}, + "mean_score": 0.25, + "average_query_total_token_usage": {"answer": 6.0}, + "queries": { + "q1": {"question_type": "fact", "score": 0.5, "status": "completed"}, + "q2": {"question_type": "fact", "score": None, "status": "candidate_failure"}, + }, + "construction_error_count": 1, + "query_error_count": 1, + }, + "two": { + "construction": {"status": "failure", "success_job_count": 0, "fail_job_count": 0}, + "mean_score": 0, + "average_query_total_token_usage": {}, + "queries": {}, + "stage": "construction", + }, + }, + } + write_json( + tmp_path / "manifest.json", + { + "status": "completed", + "mode": "scored", + "case_ids": ["one", "two"], + "query_ids": {"one": ["q1", "q2"], "two": ["q3"]}, + }, + ) + assert rebuild_summary(tmp_path) == summary + assert read_json(tmp_path / "summary.json") == summary + failure_path = tmp_path / "cases/two/failure.json" + failure = read_json(failure_path) + failure["failure_kind"] = "infra_error" + write_json(failure_path, failure) + infra = rebuild_summary(tmp_path) + assert infra["mean_query_score"] is None + assert infra["observed_mean_query_score"] == 0.5 + assert infra["cases"]["two"]["mean_score"] is None + assert infra["cases"]["one"]["mean_score"] == 0.25 + + +def test_token_averages_exclude_zero_unknown_and_judge_cost_but_count_all_returned_queries(): + assert average_tokens( + [ + { + "answer_result": {"token_usage": {"answer": {"total_tokens": 12}, "zero": {"total_tokens": 0}}}, + "judge_result": {"token_usage": {"judge": {"total_tokens": 100}}}, + }, + {"answer_result": {"token_usage": {"unknown": {"total_tokens": None}, "bad": {"total_tokens": True}}}}, + {}, + ], + ) == {"answer": 4.0} + + +def test_compact_json_round_trips_strings_and_keeps_small_leaves_inline(tmp_path): + value = {"result": {"token_usage": 0}, "unicode": ["你好", 'quote"\n'], "large": [1, 2, 3]} + path = tmp_path / "result.json" + write_json(path, value, compact_leaf_containers=True) + assert read_json(path) == value + assert '"result": {"token_usage": 0}' in path.read_text() + assert "\\u4f60" not in path.read_text() + with pytest.raises(ValueError): + write_json(path, {"score": float("nan")}, compact_leaf_containers=True) + assert json.loads(path.read_text()) == value + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stop", [RuntimeError("judge unavailable"), asyncio.CancelledError()]) +async def test_judge_interruption_preserves_all_frozen_answers_and_compacts_each_response_once(tmp_path, stop): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from deploy.benchmark_docker.pool import CaseResult + from reme_home.evaluation.component import HomeEvaluation + from reme_home.evaluation.inputs import EvaluationSpec + + case = CaseSpec( + case_id="case", + sessions=[], + queries=[QuerySpec(query_id=f"q{i}", question="?", golden_answer="a") for i in range(2)], + ) + answers = [ + { + "query_id": query.query_id, + "answer": "frozen answer", + "error": None, + "answer_result": {"answer": "frozen answer", "job_call_counts": {"agentic_answer": 2}}, + } + for query in case.queries + ] + original = deepcopy(answers) + result = CaseResult("case", build={"success": True, "jobs": []}, queries={"queries": answers}) + spec = EvaluationSpec(fail_fast=True) + await HomeEvaluation._persist_case(tmp_path, case, result, None, spec) # pylint: disable=protected-access + judge = SimpleNamespace(evaluate=AsyncMock(side_effect=stop)) + with pytest.raises(type(stop)): + await HomeEvaluation._score_case( # pylint: disable=protected-access + tmp_path, + case, + "beam", + judge, + asyncio.Semaphore(1), + spec, + answers, + ) + rows = read_json(tmp_path / "cases/case/queries/result.json")["queries"] + assert [row["answer"] for row in rows] == ["frozen answer", "frozen answer"] + assert [row["answer_result"]["job_call_counts"] for row in rows] == [{"agentic_answer": 1}] * 2 + assert answers == original + assert not (tmp_path / "cases/case/queries/answers.json").exists() + + +def test_rebuild_rejects_missing_completed_answers_without_replacing_summary(tmp_path): + write_json( + tmp_path / "manifest.json", + {"status": "completed", "case_ids": ["case"], "query_ids": {"case": ["q"]}}, + ) + write_json(tmp_path / "cases/case/memory_construction/result.json", {"status": "success", "jobs": []}) + write_json(tmp_path / "summary.json", {"preserve": "existing summary"}) + with pytest.raises(ValueError, match="incomplete query results"): + rebuild_summary(tmp_path) + assert read_json(tmp_path / "summary.json") == {"preserve": "existing summary"}