mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-16 09:21:50 +00:00
refactor(home): align evaluation summaries with dev_meta
This commit is contained in:
parent
f9918168ac
commit
95f9ea7b61
8 changed files with 877 additions and 155 deletions
|
|
@ -241,24 +241,25 @@ Results use `home/evaluations/<branch>/<commit-prefix>/<ascending-run-id>/`:
|
|||
manifest.json
|
||||
summary.json
|
||||
cases/<case-id>/
|
||||
execution.json
|
||||
memory_construction/
|
||||
result.json
|
||||
build.log
|
||||
reme_workspace.tar.gz
|
||||
reme_workspace/
|
||||
evaluation.tar.gz
|
||||
queries/
|
||||
answers.json
|
||||
result.json
|
||||
<query-id>/answer.log
|
||||
```
|
||||
|
||||
The memory tar is exported **before** queries, retains session Git checkpoints,
|
||||
and is independent of the post-query evaluation archive. Replayed runs keep an
|
||||
and is independent of the query execution state. Replayed runs keep an
|
||||
independent donor tar and inherited build log without charging construction
|
||||
tokens again. Answers are persisted before host judging. Per-query results keep
|
||||
reference/actual answers, scores, judge output, token usage and failures. Summary
|
||||
reference/actual answers, scores, compact judge output, reported answer token usage
|
||||
and failures. `queries/result.json` is the sole authoritative answer record;
|
||||
reconstructable prompts and repeated payloads are removed using the reviewed
|
||||
`dev_meta` rules. Construction results similarly compact redundant links, Job
|
||||
counts, token records and cumulative daily-index snapshots. Summary
|
||||
scores use the expected-query denominator; infrastructure/judge/artifact errors
|
||||
make the comparable mean unavailable while retaining the observed-score mean.
|
||||
The Job returns `success=false` for infrastructure/judge/artifact failures, while
|
||||
|
|
@ -273,8 +274,17 @@ their error text. Successful jobs returning empty answers still count as candida
|
|||
failures. The construction failure-rate tolerance applies only to candidate
|
||||
errors: infrastructure or unclassified failures always reject construction, even
|
||||
when another failed job was a candidate error.
|
||||
`valid_for_comparison` does not assert that an initial baseline covers the full
|
||||
dataset: the caller must also verify its code and complete case/query coverage.
|
||||
The manifest retains run identity, selection, mode and execution status. The
|
||||
summary retains only outcome counts, expected/scored query counts, score means,
|
||||
low-score cases, question-type statistics and compact case/query mappings.
|
||||
Candidate answer token summaries average only total tokens per model; detailed
|
||||
construction costs remain in construction results. Temporary post-query archives
|
||||
are discarded after extracting logs; separate execution, checksum and raw-answer
|
||||
JSON copies are not saved. See the
|
||||
[artifact guide](src/reme_home/runtime_docs/validation-artifact-guide.md) for the
|
||||
exact fields, rebuild helper and migration from the previous verbose format.
|
||||
Before comparing runs, verify completed scored execution, infrastructure outcomes,
|
||||
candidate code and complete case/query coverage.
|
||||
|
||||
Run/case/query `failure.json` files preserve diagnostics. Interrupted/fail-fast
|
||||
runs publish `partial_summary.json`, not a completed summary. New executions
|
||||
|
|
@ -292,8 +302,9 @@ returns `success=false` for candidate failures as well as infrastructure failure
|
|||
It cannot reuse construction. Case/query selections refer to the selected mode's
|
||||
dataset; omitted selections run all its cases/queries.
|
||||
Debug results retain execution counts and artifacts, but all mean scores are
|
||||
null and `valid_for_comparison=false`. `expected_scored_query_count` is zero,
|
||||
while `expected_query_count` still records the expected execution count.
|
||||
null. `scored_query_count` and `query_status_counts.unscored` are zero, while
|
||||
`expected_query_count` still records the expected execution count. Debug/scored
|
||||
mode is recorded in the manifest.
|
||||
The former `dataset_kind` input is rejected; use `debug` instead. Debug results
|
||||
must not be used to assess answer correctness or candidate quality. No real
|
||||
Docker/model evaluation is required by Home's isolated unit tests.
|
||||
|
|
|
|||
|
|
@ -7,6 +7,9 @@ from pathlib import Path, PurePosixPath
|
|||
import shutil
|
||||
import tarfile
|
||||
import tempfile
|
||||
from uuid import uuid4
|
||||
|
||||
from ..preparation.data.models import CaseSpec, QuerySpec
|
||||
|
||||
from ..workspace import _validate_directory
|
||||
|
||||
|
|
@ -32,13 +35,16 @@ def read_json(path: Path):
|
|||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def write_json(path: Path, value) -> None:
|
||||
def write_json(path: Path, value, *, compact_leaf_containers: bool = False) -> None:
|
||||
"""Atomically replace a rebuildable or progressively published result."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor, name = tempfile.mkstemp(prefix=".result-", dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
|
||||
json.dump(value, stream, ensure_ascii=False, allow_nan=False, indent=2)
|
||||
if compact_leaf_containers:
|
||||
stream.write(_dumps_compact_leaf_containers(value))
|
||||
else:
|
||||
json.dump(value, stream, ensure_ascii=False, allow_nan=False, indent=2)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
|
|
@ -47,6 +53,69 @@ def write_json(path: Path, value) -> None:
|
|||
Path(name).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _dumps_compact_leaf_containers(value) -> str:
|
||||
"""Pretty-print JSON while keeping small scalar-only containers on one line."""
|
||||
|
||||
def is_atom(item) -> bool:
|
||||
return isinstance(item, (str, int, float, bool, type(None)))
|
||||
|
||||
strings: set[str] = set()
|
||||
|
||||
def collect_strings(item) -> None:
|
||||
if isinstance(item, str):
|
||||
strings.add(item)
|
||||
elif isinstance(item, dict):
|
||||
for key, child in item.items():
|
||||
if isinstance(key, str):
|
||||
strings.add(key)
|
||||
collect_strings(child)
|
||||
elif isinstance(item, (list, tuple)):
|
||||
for child in item:
|
||||
collect_strings(child)
|
||||
|
||||
collect_strings(value)
|
||||
marker_prefix = f"\0meta-reme-singleton-{uuid4().hex}-"
|
||||
while any(item.startswith(marker_prefix) for item in strings):
|
||||
marker_prefix = f"\0meta-reme-singleton-{uuid4().hex}-"
|
||||
|
||||
replacements: dict[str, str] = {}
|
||||
|
||||
def replace_leaf_containers(item):
|
||||
if isinstance(item, dict):
|
||||
if len(item) == 1 and is_atom(next(iter(item.values()))):
|
||||
marker = f"{marker_prefix}{len(replacements)}"
|
||||
replacements[marker] = json.dumps(
|
||||
item,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
allow_nan=False,
|
||||
separators=(",", ": "),
|
||||
)
|
||||
return marker
|
||||
return {key: replace_leaf_containers(child) for key, child in item.items()}
|
||||
if isinstance(item, (list, tuple)):
|
||||
if 1 <= len(item) <= 2 and all(is_atom(child) for child in item):
|
||||
marker = f"{marker_prefix}{len(replacements)}"
|
||||
replacements[marker] = json.dumps(
|
||||
item,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
allow_nan=False,
|
||||
separators=(",", ": "),
|
||||
)
|
||||
return marker
|
||||
children = [replace_leaf_containers(child) for child in item]
|
||||
return children if isinstance(item, list) else tuple(children)
|
||||
return item
|
||||
|
||||
transformed = replace_leaf_containers(value)
|
||||
content = json.dumps(transformed, ensure_ascii=False, indent=2, sort_keys=True, allow_nan=False)
|
||||
for marker, replacement in replacements.items():
|
||||
encoded_marker = json.dumps(marker, ensure_ascii=False)
|
||||
content = content.replace(encoded_marker, replacement)
|
||||
return content
|
||||
|
||||
|
||||
def file_sha256(path: Path) -> str:
|
||||
"""Hash potentially large snapshots without loading them into memory."""
|
||||
with path.open("rb") as stream:
|
||||
|
|
@ -98,14 +167,19 @@ def average_tokens(queries: list[dict]) -> dict:
|
|||
"""Average candidate answer total tokens over returned queries, like Meta-ReMe."""
|
||||
totals = {}
|
||||
for query in queries:
|
||||
for model, usage in (query.get("answer_result") or {}).get("token_usage", {}).items():
|
||||
usage_by_model = (query.get("answer_result") or {}).get("token_usage", {})
|
||||
if not isinstance(usage_by_model, dict):
|
||||
continue
|
||||
for model, usage in usage_by_model.items():
|
||||
if not isinstance(usage, dict):
|
||||
continue
|
||||
total = usage.get("total_tokens")
|
||||
if isinstance(total, int) and not isinstance(total, bool):
|
||||
if isinstance(total, int) and not isinstance(total, bool) and total != 0:
|
||||
totals[model] = totals.get(model, 0) + total
|
||||
return {model: round(total / len(queries), 3) for model, total in totals.items()} if queries else {}
|
||||
|
||||
|
||||
def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = True) -> dict:
|
||||
def summarize(run: Path, cases: list, *, scoring_required: bool = True) -> dict:
|
||||
"""Rebuild compact optimizer evidence with Meta-ReMe's expected-query denominator."""
|
||||
rows, all_scores, query_results = {}, [], []
|
||||
case_counts = {"completed": 0, "candidate_failure": 0, "infra_error": 0}
|
||||
|
|
@ -113,18 +187,19 @@ def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = T
|
|||
query_counts = {"completed": 0, "candidate_failure": 0, "infra_error": 0}
|
||||
stages = {"construction": 0, "query": 0}
|
||||
expected = sum(len(case.queries) for case in cases)
|
||||
expected_scored = expected if scoring_required else 0
|
||||
for case in cases:
|
||||
folder = run / "cases" / case.case_id
|
||||
construction_path = folder / "memory_construction/result.json"
|
||||
folder = contained(run, f"cases/{case.case_id}", directory=True)
|
||||
construction_path = contained(folder, "memory_construction/result.json")
|
||||
construction = read_json(construction_path) if construction_path.is_file() else {}
|
||||
query_path = folder / "queries/result.json"
|
||||
query_path = contained(folder, "queries/result.json")
|
||||
queries = read_json(query_path).get("queries", []) if query_path.is_file() else []
|
||||
failure = read_json(folder / "failure.json") if (folder / "failure.json").is_file() else {}
|
||||
if not construction and not queries and not failure:
|
||||
continue
|
||||
scores = [query["score"] for query in queries if query.get("score") is not None]
|
||||
all_scores.extend(scores)
|
||||
query_results.extend(queries)
|
||||
failed = sum(bool(query.get("error")) for query in queries)
|
||||
failed = sum(query.get("error") is not None for query in queries)
|
||||
has_infra = failure.get("failure_kind") == "infra_error" or any(q.get("infrastructure_error") for q in queries)
|
||||
case_status = (
|
||||
"infra_error"
|
||||
|
|
@ -136,14 +211,16 @@ def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = T
|
|||
if construction_status == "reused":
|
||||
construction_status = "success"
|
||||
construction_counts[construction_status] += 1
|
||||
stages["construction"] += construction_status == "failure"
|
||||
jobs = [job for job in construction.get("jobs", []) if isinstance(job, dict)]
|
||||
job_results = [job["result"] if isinstance(job.get("result"), dict) else {} for job in jobs]
|
||||
construction_errors = sum(not result.get("success") for result in job_results)
|
||||
stages["construction"] += construction_status == "failure" or bool(construction_errors)
|
||||
stages["query"] += bool(failed)
|
||||
jobs = construction.get("jobs", [])
|
||||
compact_queries = {}
|
||||
for query in queries:
|
||||
query_status = (
|
||||
"completed"
|
||||
if not query.get("error")
|
||||
if query.get("error") is None
|
||||
else "infra_error" if query.get("infrastructure_error") else "candidate_failure"
|
||||
)
|
||||
query_counts[query_status] += 1
|
||||
|
|
@ -155,17 +232,19 @@ def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = T
|
|||
rows[case.case_id] = {
|
||||
"construction": {
|
||||
"status": construction_status,
|
||||
"success_job_count": sum(job.get("result", {}).get("success") is True for job in jobs),
|
||||
"fail_job_count": sum(job.get("result", {}).get("success") is not True for job in jobs),
|
||||
"success_job_count": len(jobs) - construction_errors,
|
||||
"fail_job_count": construction_errors,
|
||||
},
|
||||
"expected_query_count": len(case.queries),
|
||||
"expected_scored_query_count": len(case.queries) if scoring_required else 0,
|
||||
"scored_count": len(scores),
|
||||
"query_error_count": failed,
|
||||
"mean_score": None if has_infra or not scoring_required else sum(scores) / len(case.queries),
|
||||
"average_query_total_token_usage": average_tokens(queries),
|
||||
"queries": compact_queries,
|
||||
}
|
||||
if failure.get("stage") is not None:
|
||||
rows[case.case_id]["stage"] = failure["stage"]
|
||||
if construction_errors:
|
||||
rows[case.case_id]["construction_error_count"] = construction_errors
|
||||
if failed:
|
||||
rows[case.case_id]["query_error_count"] = failed
|
||||
groups = {}
|
||||
for query in query_results:
|
||||
groups.setdefault(str(query.get("question_type") or "unknown"), []).append(query)
|
||||
|
|
@ -178,25 +257,18 @@ def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = T
|
|||
"average_query_total_token_usage": average_tokens(queries),
|
||||
}
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"status": status,
|
||||
"case_count": len(cases),
|
||||
"expected_case_count": len(cases),
|
||||
"case_count": len(rows),
|
||||
"case_status_counts": case_counts,
|
||||
"construction_status_counts": construction_counts,
|
||||
"failure_stage_case_counts": stages,
|
||||
"expected_query_count": expected,
|
||||
"expected_scored_query_count": expected_scored,
|
||||
"scoring_required": scoring_required,
|
||||
"scored_count": len(all_scores),
|
||||
"scored_query_count": len(all_scores),
|
||||
"unscored_count": expected_scored - len(all_scores),
|
||||
"query_status_counts": {**query_counts, "unscored": expected_scored - len(all_scores)},
|
||||
"query_error_count": query_counts["candidate_failure"] + query_counts["infra_error"],
|
||||
"scoring_complete": scoring_required and len(all_scores) == expected_scored,
|
||||
"valid_for_comparison": scoring_required and status == "completed" and not case_counts["infra_error"],
|
||||
"query_status_counts": {
|
||||
**query_counts,
|
||||
"unscored": max(expected - len(all_scores), 0) if scoring_required else 0,
|
||||
},
|
||||
"mean_query_score": (
|
||||
sum(all_scores) / expected_scored if expected_scored and not case_counts["infra_error"] else None
|
||||
sum(all_scores) / expected if scoring_required and expected and not case_counts["infra_error"] else None
|
||||
),
|
||||
"observed_mean_query_score": sum(all_scores) / len(all_scores) if all_scores else None,
|
||||
"low_score_cases": [
|
||||
|
|
@ -213,6 +285,38 @@ def summarize(run: Path, cases: list, *, status: str, scoring_required: bool = T
|
|||
}
|
||||
|
||||
|
||||
def rebuild_summary(run: Path) -> dict:
|
||||
"""Rebuild compact evidence from the manifest and authoritative phase results."""
|
||||
manifest = read_json(run / "manifest.json")
|
||||
cases = [
|
||||
CaseSpec(
|
||||
case_id=case_id,
|
||||
sessions=[],
|
||||
queries=[QuerySpec(query_id=query_id, question="", golden_answer=None) for query_id in query_ids],
|
||||
)
|
||||
for case_id, query_ids in manifest["query_ids"].items()
|
||||
]
|
||||
by_id = {case.case_id: case for case in cases}
|
||||
if manifest.get("status") == "completed":
|
||||
for case_id in manifest["case_ids"]:
|
||||
folder = contained(run, f"cases/{case_id}", directory=True)
|
||||
read_json(contained(folder, "memory_construction/result.json"))
|
||||
query_path = contained(folder, "queries/result.json")
|
||||
queries = read_json(query_path)["queries"] if query_path.is_file() else []
|
||||
actual_ids = [query["query_id"] for query in queries]
|
||||
if actual_ids != manifest["query_ids"][case_id]:
|
||||
if queries or not (folder / "failure.json").is_file():
|
||||
raise ValueError(f"Completed evaluation has incomplete query results: {case_id}")
|
||||
summary = summarize(
|
||||
run,
|
||||
[by_id[case_id] for case_id in manifest["case_ids"]],
|
||||
scoring_required=manifest.get("mode") != "debug",
|
||||
)
|
||||
name = "summary.json" if manifest.get("status") == "completed" else "partial_summary.json"
|
||||
write_json(run / name, summary, compact_leaf_containers=True)
|
||||
return summary
|
||||
|
||||
|
||||
INFRA_TYPES = {
|
||||
"APIConnectionError",
|
||||
"APITimeoutError",
|
||||
|
|
|
|||
177
plugins/home/src/reme_home/evaluation/compaction.py
Normal file
177
plugins/home/src/reme_home/evaluation/compaction.py
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
"""Result compaction rules preserved from dev_meta b53048e4.
|
||||
|
||||
Keep diagnostic evidence while removing reconstructable prompts and duplicate data.
|
||||
Compaction copies inputs so scoring and failure classification retain raw responses.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from copy import deepcopy
|
||||
from pathlib import PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
|
||||
def compact_query(result: dict) -> dict:
|
||||
"""Copy and compact one authoritative query result."""
|
||||
result = deepcopy(result)
|
||||
_compact_query_result(result)
|
||||
return result
|
||||
|
||||
|
||||
def compact_construction(build: dict) -> dict:
|
||||
"""Copy and compact ordered construction jobs using the reviewed rules."""
|
||||
persisted_build = deepcopy(build)
|
||||
_compact_construction_source_conversations(persisted_build)
|
||||
_compact_construction_index_snapshots(persisted_build)
|
||||
for job in persisted_build.get("jobs", []):
|
||||
if not isinstance(job, Mapping):
|
||||
continue
|
||||
if isinstance(job, dict):
|
||||
job.pop("memory_checkpoint", None)
|
||||
result = job.get("result")
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
_compact_outer_job_call_count(job.get("job"), result)
|
||||
token_usage = result.get("token_usage")
|
||||
if not isinstance(token_usage, dict):
|
||||
continue
|
||||
for backend, usage in list(token_usage.items()):
|
||||
if isinstance(usage, Mapping) and all(
|
||||
usage.get(field) is None for field in ("input_tokens", "output_tokens", "total_tokens")
|
||||
):
|
||||
del token_usage[backend]
|
||||
if not token_usage:
|
||||
del result["token_usage"]
|
||||
elif all(
|
||||
isinstance(usage, Mapping)
|
||||
and all(usage.get(field) == 0 for field in ("input_tokens", "output_tokens", "total_tokens"))
|
||||
for usage in token_usage.values()
|
||||
):
|
||||
result["token_usage"] = 0
|
||||
return persisted_build
|
||||
|
||||
|
||||
def _compact_query_result(result: dict[str, Any]) -> None:
|
||||
"""Remove reconstructable prompts and uninformative token records before persistence."""
|
||||
|
||||
answer_result = result.get("answer_result")
|
||||
if isinstance(answer_result, dict):
|
||||
if answer_result.get("answer") == result.get("answer"):
|
||||
answer_result.pop("answer", None)
|
||||
if "error" in result and answer_result.get("error") == result.get("error"):
|
||||
answer_result.pop("error", None)
|
||||
_compact_outer_job_call_count("agentic_answer", answer_result)
|
||||
metadata = answer_result.get("metadata")
|
||||
if isinstance(metadata, dict):
|
||||
if metadata.get("query") == result.get("question"):
|
||||
metadata.pop("query", None)
|
||||
if metadata.get("response") == result.get("answer"):
|
||||
metadata.pop("response", None)
|
||||
|
||||
judge_result = result.get("judge_result")
|
||||
if isinstance(judge_result, dict):
|
||||
judge_result.pop("token_usage", None)
|
||||
if result.get("score") is not None:
|
||||
judge_result.pop("answer", None)
|
||||
_compact_outer_job_call_count("answer_judge", judge_result)
|
||||
metadata = judge_result.get("metadata")
|
||||
if isinstance(metadata, dict):
|
||||
if metadata.get("raw_answer_judgement") is not None:
|
||||
metadata.pop("answer_judgement", None)
|
||||
for field in (
|
||||
"agent_answer",
|
||||
"golden_answer",
|
||||
"llm_response",
|
||||
"probing_question",
|
||||
"query",
|
||||
"question_type",
|
||||
"rubric",
|
||||
):
|
||||
metadata.pop(field, None)
|
||||
|
||||
def compact(value: Any) -> None:
|
||||
if isinstance(value, dict):
|
||||
value.pop("sys_prompt", None)
|
||||
token_usage = value.get("token_usage")
|
||||
if isinstance(token_usage, dict):
|
||||
for backend, usage in list(token_usage.items()):
|
||||
if isinstance(usage, Mapping) and all(
|
||||
usage.get(field) is None for field in ("input_tokens", "output_tokens", "total_tokens")
|
||||
):
|
||||
del token_usage[backend]
|
||||
if not token_usage:
|
||||
del value["token_usage"]
|
||||
for child in value.values():
|
||||
compact(child)
|
||||
elif isinstance(value, list):
|
||||
for child in value:
|
||||
compact(child)
|
||||
|
||||
compact(result)
|
||||
|
||||
|
||||
def _compact_outer_job_call_count(job_name: Any, result: dict[str, Any]) -> None:
|
||||
"""Remove one call already represented by the enclosing result record."""
|
||||
|
||||
counts = result.get("job_call_counts")
|
||||
if not isinstance(job_name, str) or not isinstance(counts, dict):
|
||||
return
|
||||
own_count = counts.get(job_name)
|
||||
if isinstance(own_count, int) and not isinstance(own_count, bool) and own_count > 0:
|
||||
remaining = own_count - 1
|
||||
if remaining:
|
||||
counts[job_name] = remaining
|
||||
else:
|
||||
del counts[job_name]
|
||||
if not counts:
|
||||
del result["job_call_counts"]
|
||||
|
||||
|
||||
def _compact_construction_index_snapshots(value: Any) -> None:
|
||||
"""Replace cumulative daily-index note snapshots with their counts."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
index = value.get("index")
|
||||
if _is_daily_index_snapshot(index):
|
||||
index["notes_count"] = len(index.pop("notes"))
|
||||
for child in value.values():
|
||||
_compact_construction_index_snapshots(child)
|
||||
elif isinstance(value, list):
|
||||
for child in value:
|
||||
_compact_construction_index_snapshots(child)
|
||||
|
||||
|
||||
def _is_daily_index_snapshot(value: Any) -> bool:
|
||||
"""Recognize the stable ``refresh_day_index`` payload without assuming a configured directory name."""
|
||||
|
||||
if not isinstance(value, dict) or not isinstance(value.get("notes"), list):
|
||||
return False
|
||||
date = value.get("date")
|
||||
path = value.get("path")
|
||||
if not isinstance(date, str) or not isinstance(path, str):
|
||||
return False
|
||||
index_path = PurePosixPath(path)
|
||||
return (
|
||||
index_path.suffix == ".md"
|
||||
and index_path.stem == date
|
||||
and isinstance(value.get("created"), bool)
|
||||
and isinstance(value.get("changed"), bool)
|
||||
)
|
||||
|
||||
|
||||
def _compact_construction_source_conversations(value: Any) -> None:
|
||||
"""Drop a transcript link when its sibling session ID already identifies the same file."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
session_id = value.get("session_id")
|
||||
source_conversation = value.get("source_conversation")
|
||||
if isinstance(session_id, str) and isinstance(source_conversation, str):
|
||||
source_path = source_conversation.strip()
|
||||
if source_path.startswith("[[") and source_path[-2:] == "]]":
|
||||
source_path = source_path[2:-2]
|
||||
if PurePosixPath(source_path).name == f"{session_id}.jsonl":
|
||||
del value["source_conversation"]
|
||||
for child in value.values():
|
||||
_compact_construction_source_conversations(child)
|
||||
elif isinstance(value, list):
|
||||
for child in value:
|
||||
_compact_construction_source_conversations(child)
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
"""Home evaluation orchestration using the shared Docker pool and a host judge."""
|
||||
|
||||
import asyncio
|
||||
from dataclasses import asdict
|
||||
import os
|
||||
import shutil
|
||||
import traceback
|
||||
|
|
@ -24,6 +23,7 @@ from .artifacts import (
|
|||
failure_kind,
|
||||
)
|
||||
from .inputs import EvaluationSpec, load_inputs, reuse_inputs, snapshot_code
|
||||
from .compaction import compact_construction, compact_query
|
||||
from ..judge import HomeJudge
|
||||
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
|
||||
|
||||
|
|
@ -146,9 +146,9 @@ class HomeEvaluation(BaseComponent):
|
|||
"execution_mode": "query_only_replay" if source else "full_validation",
|
||||
"construction_source": source,
|
||||
"fingerprints": fingerprints,
|
||||
"options": spec.model_dump(mode="json"),
|
||||
"options": spec.model_dump(mode="json", exclude={"case_ids", "query_ids", "reuse_construction_from"}),
|
||||
"judge": self.judge.identity(dataset.dataset),
|
||||
"judge_model": self.judge.identity(dataset.dataset)["model"],
|
||||
"status": "running",
|
||||
"started_at": utc_now().isoformat(),
|
||||
}
|
||||
run = self._allocate(branch, commit, manifest)
|
||||
|
|
@ -214,7 +214,15 @@ class HomeEvaluation(BaseComponent):
|
|||
run / "cases" / plan.case_id / "failure.json",
|
||||
{"stage": "artifacts", "error": result.error, "failure_kind": "infra_error"},
|
||||
)
|
||||
await self._score_case(run, by_id[plan.case_id], dataset.dataset, judge, semaphore, spec)
|
||||
await self._score_case(
|
||||
run,
|
||||
by_id[plan.case_id],
|
||||
dataset.dataset,
|
||||
judge,
|
||||
semaphore,
|
||||
spec,
|
||||
(result.queries or {}).get("queries", []),
|
||||
)
|
||||
|
||||
results = await BenchmarkDockerPool(factory, concurrency=spec.concurrency).run(
|
||||
plans,
|
||||
|
|
@ -224,9 +232,6 @@ class HomeEvaluation(BaseComponent):
|
|||
)
|
||||
# Pool teardown can attach late cleanup failures after callbacks.
|
||||
for result in results:
|
||||
execution = asdict(result)
|
||||
execution["workspace_snapshot"] = str(result.workspace_snapshot) if result.workspace_snapshot else None
|
||||
write_json(run / "cases" / result.case_id / "execution.json", execution)
|
||||
if result.error:
|
||||
failure = run / "cases" / result.case_id / "failure.json"
|
||||
if not failure.exists():
|
||||
|
|
@ -236,22 +241,24 @@ class HomeEvaluation(BaseComponent):
|
|||
if previous.get("error") != result.error:
|
||||
previous.update(cleanup_error=result.error, failure_kind="infra_error")
|
||||
write_json(failure, previous)
|
||||
summary = summarize(run, cases, status="completed", scoring_required=not spec.debug)
|
||||
summary.update(
|
||||
execution_complete=True,
|
||||
execution_mode=manifest["execution_mode"],
|
||||
)
|
||||
summary = summarize(run, cases, scoring_required=not spec.debug)
|
||||
manifest.update(status="completed", finished_at=utc_now().isoformat())
|
||||
write_json(run / "manifest.json", manifest)
|
||||
write_json(run / "summary.json", summary)
|
||||
write_json(run / "summary.json", summary, compact_leaf_containers=True)
|
||||
return {
|
||||
"result_dir": str(run),
|
||||
"commit_sha": commit,
|
||||
"execution_mode": manifest["execution_mode"],
|
||||
"status": "completed",
|
||||
**{key: value for key, value in summary.items() if key != "cases"},
|
||||
}
|
||||
except BaseException as error:
|
||||
if run is not None:
|
||||
manifest.update(
|
||||
status="cancelled" if isinstance(error, asyncio.CancelledError) else "failed",
|
||||
finished_at=utc_now().isoformat(),
|
||||
)
|
||||
write_json(run / "manifest.json", manifest)
|
||||
write_json(
|
||||
run / "failure.json",
|
||||
{
|
||||
|
|
@ -264,11 +271,9 @@ class HomeEvaluation(BaseComponent):
|
|||
partial = summarize(
|
||||
run,
|
||||
cases,
|
||||
status="cancelled" if isinstance(error, asyncio.CancelledError) else "failed",
|
||||
scoring_required=not spec.debug,
|
||||
)
|
||||
partial["execution_complete"] = False
|
||||
write_json(run / "partial_summary.json", partial)
|
||||
write_json(run / "partial_summary.json", partial, compact_leaf_containers=True)
|
||||
if isinstance(error, asyncio.CancelledError) or run is None:
|
||||
raise
|
||||
raise RuntimeError(f"Evaluation failed; artifacts: {run}: {error}") from error
|
||||
|
|
@ -277,10 +282,25 @@ class HomeEvaluation(BaseComponent):
|
|||
async def _persist_case(run, case, result, source, spec):
|
||||
folder = run / "cases" / case.case_id
|
||||
memory = folder / "memory_construction"
|
||||
execution = asdict(result)
|
||||
execution["workspace_snapshot"] = str(result.workspace_snapshot) if result.workspace_snapshot else None
|
||||
write_json(folder / "execution.json", execution)
|
||||
write_json(folder / "queries/answers.json", result.queries or {"queries": []})
|
||||
# Freeze every returned answer in the sole authoritative query file
|
||||
# before extraction or judging can fail. Unjudged answers stay present.
|
||||
by_id = {row.get("query_id"): row for row in (result.queries or {}).get("queries", [])}
|
||||
rows = [
|
||||
compact_query(
|
||||
{
|
||||
**by_id[query.query_id],
|
||||
"query_id": query.query_id,
|
||||
"question": query.question,
|
||||
"golden_answer": query.golden_answer,
|
||||
"question_type": query.metadata.get("question_type"),
|
||||
"score": None,
|
||||
"judge_result": None,
|
||||
},
|
||||
)
|
||||
for query in case.queries
|
||||
if query.query_id in by_id
|
||||
]
|
||||
write_json(folder / "queries/result.json", {"queries": rows}, compact_leaf_containers=True)
|
||||
if result.error:
|
||||
stage = result.failure_stage or "execution"
|
||||
exception = result.exception
|
||||
|
|
@ -309,7 +329,7 @@ class HomeEvaluation(BaseComponent):
|
|||
archive = memory / "reme_workspace.tar.gz"
|
||||
if archive.is_file():
|
||||
construction["workspace_sha256"] = await blocking(file_sha256, archive)
|
||||
write_json(memory / "result.json", construction)
|
||||
write_json(memory / "result.json", compact_construction(construction), compact_leaf_containers=True)
|
||||
archive = memory / "reme_workspace.tar.gz"
|
||||
if archive.is_file():
|
||||
await blocking(inspect_archive, archive, spec.max_artifact_bytes, memory / "reme_workspace")
|
||||
|
|
@ -325,25 +345,21 @@ class HomeEvaluation(BaseComponent):
|
|||
target = folder / "queries" / query.query_id / "answer.log"
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
await blocking(shutil.copyfile, log, target)
|
||||
# Do not merge the container's queries tree: it could overwrite
|
||||
# host-frozen answers.json or host-only judge results. The complete
|
||||
# untrusted tree remains available in evaluation.tar.gz for inspection.
|
||||
# Container results cannot overwrite host-frozen answers or scores.
|
||||
await blocking(shutil.rmtree, extracted)
|
||||
integrity = {}
|
||||
for relative in ("memory_construction/reme_workspace.tar.gz", "evaluation.tar.gz", "full.tar.gz"):
|
||||
artifact = folder / relative
|
||||
if artifact.is_file():
|
||||
integrity[relative] = await blocking(file_sha256, artifact)
|
||||
write_json(folder / "artifacts.json", {"sha256": integrity})
|
||||
evaluation.unlink()
|
||||
|
||||
@staticmethod
|
||||
async def _score_case(run, case, dataset, judge, semaphore, spec):
|
||||
async def _score_case(run, case, dataset, judge, semaphore, spec, answers):
|
||||
folder = run / "cases" / case.case_id
|
||||
answers = read_json(folder / "queries/answers.json").get("queries", [])
|
||||
by_id = {row.get("query_id"): row for row in answers}
|
||||
rows = []
|
||||
rows = read_json(folder / "queries/result.json").get("queries", [])
|
||||
positions = {row["query_id"]: index for index, row in enumerate(rows)}
|
||||
case_failure = read_json(folder / "failure.json") if (folder / "failure.json").exists() else {}
|
||||
for query in case.queries:
|
||||
raw = by_id.get(query.query_id, {})
|
||||
if not raw and case_failure:
|
||||
continue # Construction failures leave queries unassessed.
|
||||
row = {
|
||||
**raw,
|
||||
"query_id": query.query_id,
|
||||
|
|
@ -353,10 +369,9 @@ class HomeEvaluation(BaseComponent):
|
|||
"score": None,
|
||||
"judge_result": None,
|
||||
}
|
||||
if not isinstance(row.get("answer"), str) or not row["answer"].strip() or row.get("error"):
|
||||
if not isinstance(row.get("answer"), str) or not row["answer"].strip() or row.get("error") is not None:
|
||||
row["error"] = row.get("error") or "No successful answer (construction or execution failed)"
|
||||
row["failure_stage"] = "answer"
|
||||
case_failure = read_json(folder / "failure.json") if (folder / "failure.json").exists() else {}
|
||||
row["failure_kind"] = (
|
||||
failure_kind(row["exception"], "answer")
|
||||
if row.get("exception")
|
||||
|
|
@ -376,10 +391,14 @@ class HomeEvaluation(BaseComponent):
|
|||
failure_kind="infra_error",
|
||||
infrastructure_error=True,
|
||||
)
|
||||
rows.append(row)
|
||||
compact = compact_query(row)
|
||||
if query.query_id in positions:
|
||||
rows[positions[query.query_id]] = compact
|
||||
else:
|
||||
rows.append(compact)
|
||||
# Publish each score; a later judge failure cannot erase frozen answers.
|
||||
write_json(folder / "queries/result.json", {"queries": rows})
|
||||
if row.get("error"):
|
||||
write_json(folder / "queries" / query.query_id / "failure.json", row)
|
||||
write_json(folder / "queries/result.json", {"queries": rows}, compact_leaf_containers=True)
|
||||
if row.get("error") is not None:
|
||||
write_json(folder / "queries" / query.query_id / "failure.json", compact, compact_leaf_containers=True)
|
||||
if spec.fail_fast:
|
||||
raise RuntimeError(f"Query {case.case_id}/{query.query_id} failed: {row['error']}")
|
||||
|
|
|
|||
|
|
@ -213,7 +213,7 @@ def reuse_inputs(home: Path, spec: EvaluationSpec, cases: list, fingerprints: di
|
|||
raise ValueError("reuse_construction_from must be inside home/evaluations")
|
||||
manifest = read_json(source / "manifest.json")
|
||||
summary = read_json(source / "summary.json")
|
||||
if manifest.get("dataset_kind") != "training-set" or summary.get("status") != "completed":
|
||||
if manifest.get("dataset_kind") != "training-set" or manifest.get("status", summary.get("status")) != "completed":
|
||||
raise ValueError("Reuse requires a completed training-set evaluation")
|
||||
for key in ("dataset", "construction_config"):
|
||||
if manifest.get("fingerprints", {}).get(key) != fingerprints[key]:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
`home/evaluations/<branch>/<commit-prefix>/<run-id>/` inside the outer ReMe
|
||||
workspace. Commit prefixes start at seven characters and extend on collision.
|
||||
Run IDs increase within that commit directory; prior results are preserved.
|
||||
Result compaction and summary fields follow the reviewed `dev_meta` format.
|
||||
|
||||
## Saved files
|
||||
|
||||
|
|
@ -11,65 +12,111 @@ Paths in this table are relative to one run directory:
|
|||
|
||||
| Path | Meaning |
|
||||
| --- | --- |
|
||||
| `manifest.json` | Candidate commit, case/query selection, execution mode, input/configuration fingerprints, host judge identity, and run status. |
|
||||
| `summary.json` | Completed run summary: aggregate scores, expected/scored counts, case/query outcomes, low-score cases, and comparison validity. |
|
||||
| `partial_summary.json` | Partial results after an interrupted or failed run; not a completed baseline. |
|
||||
| `manifest.json` | Candidate commit, selected case/query IDs, dataset, execution mode, input/configuration fingerprints, host judge identity, options, timestamps and run status. Retained for reproducibility and rebuilding summaries; identity is not repeated in the summary. |
|
||||
| `summary.json` | Rebuildable evidence from a completed execution. Its fields are described below. |
|
||||
| `partial_summary.json` | Available evidence after cancellation or failure; not a completed baseline. |
|
||||
| `failure.json` | Run-level failure details, when present. |
|
||||
| `cases/<case-id>/execution.json` | Docker pool execution outcome, including execution/cleanup failures. |
|
||||
| `cases/<case-id>/artifacts.json` | Checksums of exported archives. |
|
||||
| `cases/<case-id>/memory_construction/result.json` | Construction outcome and ordered Job results, including available errors, metadata, and token usage. |
|
||||
| `cases/<case-id>/memory_construction/build.log` | Construction log. Replayed runs inherit the donor log. |
|
||||
| `cases/<case-id>/memory_construction/reme_workspace.tar.gz` | Workspace snapshot exported before answering queries; the donor for construction reuse. |
|
||||
| `cases/<case-id>/memory_construction/result.json` | Ordered construction Job results with success, errors, metadata, answers, reported token usage and inner Job call counts. Replays record their source and snapshot checksum without charging construction again. |
|
||||
| `cases/<case-id>/memory_construction/build.log` | Construction log. Replays inherit the donor log. |
|
||||
| `cases/<case-id>/memory_construction/reme_workspace.tar.gz` | Checksum-verified workspace snapshot exported before answering; the donor for construction reuse. |
|
||||
| `cases/<case-id>/memory_construction/reme_workspace/` | Extracted construction workspace, including memory files and session Git checkpoints when exported. |
|
||||
| `cases/<case-id>/evaluation.tar.gz` | Exported post-query evaluation artifacts. |
|
||||
| `cases/<case-id>/queries/answers.json` | Raw answers saved by the host before judging and artifact extraction. |
|
||||
| `cases/<case-id>/queries/result.json` | Per-query question, reference and actual answer, score, answer response, judge response, and failure information. |
|
||||
| `cases/<case-id>/queries/result.json` | The sole authoritative query record: question, reference and actual answer, question type, score, compact answer/judge responses and failures. All returned answers are frozen here before extraction and host judging; scores are published progressively. |
|
||||
| `cases/<case-id>/queries/<query-id>/answer.log` | Query execution log, when available. |
|
||||
| Case/query `failure.json` | Failure stage and diagnostics, when present. |
|
||||
| `cases/<case-id>/full.tar.gz` | Best-effort sandbox diagnostics for execution failures, when available. |
|
||||
|
||||
Exported files depend on how far execution progressed. Infrastructure failures
|
||||
and timeouts may prevent logs or full workspace archives from being recovered.
|
||||
Inspect failure records before assuming that an absent artifact means the
|
||||
candidate produced no memory or answer.
|
||||
Successful exports retain query logs, then discard the temporary post-query
|
||||
`evaluation.tar.gz`. There is no separate `execution.json`, `artifacts.json`,
|
||||
`queries/answers.json` or per-query `result.json`. A failed extraction can leave
|
||||
its archive for diagnosis. Exported files depend on execution progress; inspect
|
||||
failure records before interpreting an absent workspace or log.
|
||||
|
||||
Query compaction removes reconstructable `sys_prompt` values, duplicate nested
|
||||
answers/questions/errors, the enclosing Job's one call, judge token usage,
|
||||
duplicate parsed judgements when raw judgements exist, and repeated judge inputs.
|
||||
Distinct errors, raw judgements, rubric assessments and reported answer token
|
||||
metrics remain. Unreported token backends are omitted.
|
||||
|
||||
Construction compaction removes checkpoint descriptors already available in Git,
|
||||
one enclosing Job call, unreported token backends, and transcript links identified
|
||||
by a sibling session ID. Recognized cumulative daily-index `notes` snapshots become
|
||||
`notes_count`; unrelated plugin payloads remain. All-zero construction token
|
||||
records become `0`. Detailed construction costs stay in this case artifact.
|
||||
Small scalar containers are written on one line, following `dev_meta` formatting.
|
||||
|
||||
## Summary fields
|
||||
|
||||
The run summary contains only:
|
||||
|
||||
- `case_count`, `case_status_counts`, `construction_status_counts`, and
|
||||
`failure_stage_case_counts` (construction/query; tolerated construction failures
|
||||
still count as construction errors).
|
||||
- `expected_query_count`, `scored_query_count`, and `query_status_counts`
|
||||
(completed, candidate failure, infrastructure error, unscored).
|
||||
- `mean_query_score`, `observed_mean_query_score`, `low_score_cases`,
|
||||
`question_type_statistics`, and `cases`.
|
||||
|
||||
Each `cases[case_id]` contains construction `status`, `success_job_count` and
|
||||
`fail_job_count`; `mean_score`; `average_query_total_token_usage`; and a `queries`
|
||||
map. Each query entry contains only `question_type`, `score` and `status`.
|
||||
`stage`, `construction_error_count` and `query_error_count` appear only when
|
||||
applicable. There is no redundant case status or per-case coverage accounting.
|
||||
Questions, answers, errors, Job details and construction token usage stay in
|
||||
phase artifacts, outside the summary.
|
||||
|
||||
Token summaries contain only average candidate answer **total** tokens per model,
|
||||
divided by all returned queries, including failures and queries without reported
|
||||
usage. Zero and unknown token totals are omitted. Judge/construction costs and
|
||||
input/output token breakdowns are not aggregated. Question-type statistics contain
|
||||
query count, average of available scores (rounded to three decimals), and the same
|
||||
answer token averages across cases. Missing/empty types form `unknown`.
|
||||
|
||||
Candidate failures count as zero in the expected-query score denominator.
|
||||
Construction failures leave queries unassessed rather than inventing query
|
||||
failures. Infrastructure, artifact, cleanup or judge errors make the comparable
|
||||
run mean unavailable; `observed_mean_query_score` averages available scores only.
|
||||
`low_score_cases` contains `{case_id: mean_score}` for cases with at least one
|
||||
scored query below `1`, sorted by mean then case ID; unavailable case means are
|
||||
omitted.
|
||||
|
||||
Run status and debug/scored mode are in the manifest. Debug runs skip judging;
|
||||
run and case means are null, `scored_query_count` and the unscored count are zero,
|
||||
while `expected_query_count` records execution coverage. They cannot measure
|
||||
answer quality. Only compare completed scored runs after checking infrastructure
|
||||
outcomes, expected coverage and the candidate selection.
|
||||
|
||||
To rebuild the appropriate summary from a run's manifest and phase results:
|
||||
|
||||
```python
|
||||
from pathlib import Path
|
||||
from reme_home.evaluation.artifacts import rebuild_summary
|
||||
|
||||
summary = rebuild_summary(Path("/absolute/path/to/run"))
|
||||
```
|
||||
|
||||
New results omit the former `schema_version`, `status`, `execution_complete`,
|
||||
`execution_mode`, `expected_case_count`, `expected_scored_query_count`,
|
||||
`scoring_required`, `scoring_complete`, `valid_for_comparison`, `scored_count`,
|
||||
`unscored_count`, and top-level `query_error_count` from summaries. Use manifest
|
||||
identity/status/mode and the retained counts instead. The Job returns the run
|
||||
summary without `cases`, plus `result_dir`, `commit_sha`, `execution_mode` and
|
||||
execution `status`. Existing result directories are not rewritten automatically.
|
||||
|
||||
## Investigating a result
|
||||
|
||||
1. Read the manifest to confirm candidate commit, selected cases/queries, judge,
|
||||
and whether construction was built or reused.
|
||||
2. Read the summary, then inspect `queries/result.json` for an affected case.
|
||||
Compare the actual answer with the reference and judge response.
|
||||
3. Use `answer.log` to trace retrieval and answer generation. Raw
|
||||
`queries/answers.json` remains available if host judging failed.
|
||||
4. Inspect `memory_construction/result.json` and `build.log` for construction
|
||||
failures or unexpected Job behavior.
|
||||
5. Inspect the extracted pre-query workspace. Its configured memory/session
|
||||
directories and Git history show memory changes at construction checkpoints.
|
||||
Use read-only Git commands such as `git log`, `git show`, and `git diff` in
|
||||
that exported repository. Directory names depend on the runtime configuration.
|
||||
1. Read the manifest for candidate, selection, mode, judge and construction source.
|
||||
2. Read the summary, then an affected case's `queries/result.json`. Compare the
|
||||
answer, reference and judge assessment. Frozen answers survive judge failures.
|
||||
3. Read `answer.log` for retrieval/answer execution; the host judge is outside it.
|
||||
4. Read construction `result.json` and `build.log` for Job failures and costs.
|
||||
5. Inspect the pre-query workspace and its Git history with read-only `git log`,
|
||||
`git show` and `git diff` to trace session memory checkpoints. Memory/session
|
||||
directory names depend on runtime configuration.
|
||||
|
||||
Memory construction and query answering use different snapshots. Use the
|
||||
pre-query memory tar to investigate construction or choose a reuse donor; the
|
||||
post-query archive may contain changes made while answering.
|
||||
|
||||
## Scores and comparisons
|
||||
|
||||
Candidate failures count as zero in the expected-query denominator.
|
||||
Infrastructure, artifact, or judge errors make the comparable mean unavailable;
|
||||
`observed_mean_query_score` reports the available scored observations separately.
|
||||
Failure artifacts retain structured exception types, HTTP status codes and cause
|
||||
chains from `Response.metadata["reme.job_error"]`. Infrastructure failures in any
|
||||
construction job or exception cause invalidate the comparison. Failed jobs without
|
||||
structured exception details are also treated as infrastructure errors; their
|
||||
text alone cannot establish a candidate failure. The construction failure-rate
|
||||
tolerance applies only to classified candidate errors.
|
||||
Check `valid_for_comparison`, failure details, and coverage before comparing runs.
|
||||
That flag alone does not establish a complete initial baseline: also verify the
|
||||
initial candidate commit and full training case/query coverage.
|
||||
|
||||
Debug runs (`debug=true`) test execution without host judging and cannot measure
|
||||
answer quality. Run and case mean scores are null. `expected_query_count` counts
|
||||
queries expected to execute, while `expected_scored_query_count` is zero; no
|
||||
queries are awaiting a score. Debug results are always invalid for comparison,
|
||||
including partial or failed runs.
|
||||
Construction reuse does not reuse answers or scores: the current candidate
|
||||
answers again and the host judge scores those new answers.
|
||||
Failure records preserve structured exception types, HTTP status codes and cause
|
||||
chains from `Response.metadata["reme.job_error"]`. Network/provider failures in
|
||||
any construction Job or exception cause invalidate comparison. Failed Jobs without
|
||||
structured exception details are treated as infrastructure errors. Construction
|
||||
failure-rate tolerance applies only to classified candidate errors.
|
||||
Construction reuse answers and judges again with the current candidate; it does
|
||||
not reuse donor answers or scores.
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ class FakeWorker:
|
|||
files = {
|
||||
"reme_workspace/daily/note.md": "after-answer",
|
||||
"manifest.json": "{}",
|
||||
"queries/answers.json": '{"queries": [{"query_id": "fact:1", "answer": "forged"}]}',
|
||||
"queries/result.json": '{"queries": [{"query_id": "fact:1", "answer": "forged"}]}',
|
||||
}
|
||||
for query in self.queries:
|
||||
files[f"queries/{query.query_id}/answer.log"] = "answer log"
|
||||
|
|
@ -175,17 +175,22 @@ async def test_all_cases_and_artifacts_via_real_job(app, prepared):
|
|||
response = await app.run_job("home_evaluate", concurrency=1)
|
||||
assert response.success, response.answer
|
||||
result = response.answer
|
||||
assert result["expected_case_count"] == result["scored_count"] == 2
|
||||
assert result["case_count"] == result["scored_query_count"] == 2
|
||||
assert result["mean_query_score"] == result["observed_mean_query_score"] == 0.5
|
||||
run = Path(result["result_dir"])
|
||||
manifest = json.loads((run / "manifest.json").read_text())
|
||||
assert manifest["case_ids"] == ["2", "1"]
|
||||
assert {"case_ids", "query_ids", "reuse_construction_from"}.isdisjoint(manifest["options"])
|
||||
assert "judge_model" not in manifest
|
||||
for case_id in ["1", "2"]:
|
||||
folder = run / "cases" / case_id
|
||||
assert (folder / "memory_construction/reme_workspace/daily/note.md").read_text() == case_id
|
||||
assert (folder / "memory_construction/build.log").is_file()
|
||||
assert (folder / "queries/fact:1/answer.log").is_file()
|
||||
assert (folder / "evaluation.tar.gz").is_file()
|
||||
assert not (folder / "evaluation.tar.gz").exists()
|
||||
assert not (folder / "execution.json").exists()
|
||||
assert not (folder / "artifacts.json").exists()
|
||||
assert not (folder / "queries/answers.json").exists()
|
||||
assert not (folder / "failure.json").exists()
|
||||
row = json.loads((folder / "queries/result.json").read_text())["queries"][0]
|
||||
assert row["golden_answer"] == "hello" and row["score"] == 0.5
|
||||
|
|
@ -278,11 +283,19 @@ async def test_build_failure_retains_diagnostics_and_other_cases(app, prepared,
|
|||
response = await app.run_job("home_evaluate")
|
||||
assert response.success, response.answer
|
||||
run = Path(response.answer["result_dir"])
|
||||
assert response.answer["scored_count"] == 1
|
||||
assert response.answer["scored_query_count"] == 1
|
||||
assert response.answer["mean_query_score"] == 0.25
|
||||
assert (run / "cases/1/full.tar.gz").is_file()
|
||||
assert (run / "cases/1/memory_construction/build.log").is_file()
|
||||
assert (run / "cases/1/failure.json").is_file()
|
||||
summary = json.loads((run / "summary.json").read_text())
|
||||
assert summary["cases"]["1"]["queries"] == {}
|
||||
assert summary["query_status_counts"] == {
|
||||
"completed": 1,
|
||||
"candidate_failure": 0,
|
||||
"infra_error": 0,
|
||||
"unscored": 1,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -295,8 +308,8 @@ async def test_judge_failure_preserves_answers_and_fail_fast_result(app, prepare
|
|||
response = await app.run_job("home_evaluate", case_ids=["1"])
|
||||
assert not response.success
|
||||
run = Path(response.answer["result_dir"])
|
||||
assert not response.answer["scoring_complete"]
|
||||
assert json.loads((run / "cases/1/queries/answers.json").read_text())["queries"][0]["answer"] == "hello"
|
||||
assert response.answer["query_status_counts"]["unscored"] == 1
|
||||
assert json.loads((run / "cases/1/queries/result.json").read_text())["queries"][0]["answer"] == "hello"
|
||||
assert (run / "cases/1/queries/fact:1/failure.json").is_file()
|
||||
failed = await app.run_job("home_evaluate", fail_fast=True)
|
||||
assert not failed.success
|
||||
|
|
@ -317,6 +330,8 @@ async def test_cancellation_closes_pool_and_retains_partial_run(app, prepared, m
|
|||
await task
|
||||
assert all(worker.closed for worker in factory.workers)
|
||||
assert list((prepared / "evaluations").glob("*/*/*/failure.json"))
|
||||
manifest = json.loads(next((prepared / "evaluations").glob("*/*/*/manifest.json")).read_text())
|
||||
assert manifest["status"] == "cancelled"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -324,17 +339,17 @@ async def test_debug_skips_judge_and_invalid_export_keeps_answer(app, prepared,
|
|||
response = await app.run_job("home_evaluate", debug=True)
|
||||
assert response.success, response.answer
|
||||
assert app.context.components["agent_wrapper"]["judge"].reply.await_count == 0
|
||||
assert not response.answer["scoring_required"]
|
||||
assert response.answer["scored_query_count"] == 0
|
||||
monkeypatch.setattr(FakeFactory, "bad_export", True)
|
||||
response = await app.run_job("home_evaluate", case_ids=["1"])
|
||||
assert not response.success
|
||||
run = Path(response.answer["result_dir"])
|
||||
assert (run / "cases/1/queries/answers.json").is_file()
|
||||
assert response.answer["scored_count"] == 1
|
||||
assert (run / "cases/1/queries/result.json").is_file()
|
||||
assert response.answer["scored_query_count"] == 1
|
||||
assert not list(prepared.rglob("escaped"))
|
||||
assert "artifact" in (run / "cases/1/failure.json").read_text()
|
||||
assert response.answer["mean_query_score"] is None
|
||||
assert not response.answer["valid_for_comparison"]
|
||||
assert response.answer["case_status_counts"]["infra_error"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -358,13 +373,11 @@ async def test_mode_controls_data_scoring_and_fail_fast_defaults(app, prepared,
|
|||
assert manifest["mode"] == ("debug" if debug else "scored")
|
||||
assert set(summary["cases"]) == ({"debug-1", "debug-2"} if debug else {"1", "2"})
|
||||
assert summary["expected_query_count"] == 2
|
||||
assert summary["expected_scored_query_count"] == (0 if debug else 2)
|
||||
assert summary["scoring_required"] is not debug
|
||||
assert summary["scoring_complete"] is not debug
|
||||
assert summary["valid_for_comparison"] is not debug
|
||||
assert summary["scored_query_count"] == (0 if debug else 2)
|
||||
assert manifest["status"] == "completed"
|
||||
assert summary["mean_query_score"] == (None if debug else 0.5)
|
||||
assert summary["observed_mean_query_score"] == (None if debug else 0.5)
|
||||
assert summary["unscored_count"] == 0
|
||||
assert summary["query_status_counts"]["unscored"] == 0
|
||||
for case in summary["cases"].values():
|
||||
assert case["mean_score"] == (None if debug else 0.5)
|
||||
assert app.context.components["agent_wrapper"]["judge"].reply.await_count == (0 if debug else 2)
|
||||
|
|
@ -390,11 +403,11 @@ async def test_debug_empty_answers_fail_without_scoring(app, prepared, monkeypat
|
|||
summary = json.loads((run / ("partial_summary.json" if fail_fast else "summary.json")).read_text())
|
||||
assert summary["mean_query_score"] is None
|
||||
assert summary["observed_mean_query_score"] is None
|
||||
assert summary["expected_scored_query_count"] == 0
|
||||
assert not summary["valid_for_comparison"]
|
||||
assert summary["scored_query_count"] == 0
|
||||
assert summary["query_status_counts"]["unscored"] == 0
|
||||
assert summary["case_status_counts"]["candidate_failure"] == 1
|
||||
assert summary["cases"]["debug-1"]["mean_score"] is None
|
||||
assert (run / "cases/debug-1/queries/answers.json").is_file()
|
||||
assert (run / "cases/debug-1/queries/result.json").is_file()
|
||||
assert app.context.components["agent_wrapper"]["judge"].reply.await_count == 0
|
||||
|
||||
|
||||
|
|
@ -503,7 +516,8 @@ async def test_lme_evaluation_calls_host_judge_job(app, prepared, monkeypatch):
|
|||
assert wrapper.reply.await_count == 2
|
||||
run = Path(response.answer["result_dir"])
|
||||
result = json.loads((run / "cases/q1/queries/result.json").read_text())["queries"][0]
|
||||
assert result["judge_result"]["metadata"]["answer_judgement"] == "yes"
|
||||
assert result["judge_result"]["metadata"]["raw_answer_judgement"] == "yes"
|
||||
assert "answer_judgement" not in result["judge_result"]["metadata"]
|
||||
assert result["score"] == 1
|
||||
manifest = json.loads((run / "manifest.json").read_text())
|
||||
assert manifest["judge"]["plugin"] == "lme-judge"
|
||||
|
|
@ -521,7 +535,7 @@ async def test_invalid_lme_judge_is_infra_error_after_one_retry(app, prepared):
|
|||
assert response.answer["mean_query_score"] is None
|
||||
assert wrapper.reply.await_count == 2
|
||||
run = Path(response.answer["result_dir"])
|
||||
assert (run / "cases/q1/queries/answers.json").is_file()
|
||||
assert (run / "cases/q1/queries/result.json").is_file()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -680,7 +694,6 @@ async def test_real_job_errors_reach_worker_and_home_summary(app, prepared, monk
|
|||
code_failure = kind == "code"
|
||||
assert response.success is code_failure
|
||||
summary = response.answer
|
||||
assert summary["valid_for_comparison"] is code_failure
|
||||
assert summary["case_status_counts"]["infra_error"] == (0 if code_failure else 1)
|
||||
assert summary["mean_query_score"] == (
|
||||
0.5 if code_failure and phase == "construction" else 0 if code_failure else None
|
||||
|
|
@ -692,3 +705,21 @@ async def test_real_job_errors_reach_worker_and_home_summary(app, prepared, monk
|
|||
if kind != "legacy":
|
||||
assert failure["exception"] == result["exception"]
|
||||
assert all(worker.closed for worker in FakeFactory.instances[-1].workers)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_late_cleanup_failure_survives_without_duplicate_execution_record(app, prepared, monkeypatch):
|
||||
async def close(worker):
|
||||
worker.closed = True
|
||||
raise RuntimeError("late cleanup failure")
|
||||
|
||||
monkeypatch.setattr(FakeWorker, "close", close)
|
||||
response = await app.run_job("home_evaluate", case_ids=["1"])
|
||||
assert not response.success
|
||||
assert response.answer["mean_query_score"] is None
|
||||
assert response.answer["observed_mean_query_score"] == 0.5
|
||||
folder = Path(response.answer["result_dir"]) / "cases/1"
|
||||
failure = json.loads((folder / "failure.json").read_text())
|
||||
assert failure["stage"] == "cleanup" and failure["failure_kind"] == "infra_error"
|
||||
assert "late cleanup failure" in failure["error"]
|
||||
assert not (folder / "execution.json").exists()
|
||||
|
|
|
|||
333
plugins/home/tests/test_evaluation_artifacts.py
Normal file
333
plugins/home/tests/test_evaluation_artifacts.py
Normal file
|
|
@ -0,0 +1,333 @@
|
|||
"""Reviewed dev_meta result content, compact statistics and rebuildability."""
|
||||
|
||||
# pylint: disable=missing-function-docstring
|
||||
|
||||
import asyncio
|
||||
from copy import deepcopy
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from reme_home.evaluation.artifacts import average_tokens, read_json, rebuild_summary, summarize, write_json
|
||||
from reme_home.evaluation.compaction import compact_construction, compact_query
|
||||
from reme_home.preparation.data.models import CaseSpec, QuerySpec
|
||||
|
||||
|
||||
def test_query_compaction_preserves_evidence_without_duplicate_prompts_or_costs():
|
||||
raw = {
|
||||
"question": "What happened?",
|
||||
"golden_answer": "An event.",
|
||||
"answer": "The event happened.",
|
||||
"score": 1.0,
|
||||
"error": None,
|
||||
"answer_result": {
|
||||
"success": True,
|
||||
"answer": "The event happened.",
|
||||
"error": None,
|
||||
"job_call_counts": {"agentic_answer": 2, "search": 3},
|
||||
"token_usage": {
|
||||
"answer": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13},
|
||||
"empty": {"input_tokens": None, "output_tokens": None, "total_tokens": None},
|
||||
},
|
||||
"metadata": {
|
||||
"query": "What happened?",
|
||||
"response": "The event happened.",
|
||||
"query_time": "2026-09-12T10:00:00",
|
||||
"sys_prompt": "reconstructable prompt",
|
||||
"nested": [{"sys_prompt": "nested prompt", "evidence": "keep"}],
|
||||
},
|
||||
},
|
||||
"judge_result": {
|
||||
"success": True,
|
||||
"answer": "yes",
|
||||
"job_call_counts": {"answer_judge": 1},
|
||||
"token_usage": {"judge": {"total_tokens": 100}},
|
||||
"metadata": {
|
||||
"answer_judgement": "yes",
|
||||
"raw_answer_judgement": "Yes, the answer is correct.",
|
||||
"agent_answer": "The event happened.",
|
||||
"golden_answer": "An event.",
|
||||
"llm_response": "The event happened.",
|
||||
"probing_question": "What happened?",
|
||||
"query": "What happened?",
|
||||
"question_type": "fact",
|
||||
"rubric": ["An event"],
|
||||
},
|
||||
},
|
||||
}
|
||||
original = deepcopy(raw)
|
||||
result = compact_query(raw)
|
||||
assert raw == original
|
||||
assert result["answer_result"] == {
|
||||
"success": True,
|
||||
"job_call_counts": {"agentic_answer": 1, "search": 3},
|
||||
"token_usage": {"answer": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13}},
|
||||
"metadata": {"query_time": "2026-09-12T10:00:00", "nested": [{"evidence": "keep"}]},
|
||||
}
|
||||
assert result["judge_result"] == {
|
||||
"success": True,
|
||||
"metadata": {"raw_answer_judgement": "Yes, the answer is correct."},
|
||||
}
|
||||
assert result["question"] == raw["question"]
|
||||
assert result["golden_answer"] == raw["golden_answer"]
|
||||
assert result["answer"] == raw["answer"]
|
||||
|
||||
|
||||
def test_distinct_failure_details_and_failed_judge_response_are_retained():
|
||||
raw = {
|
||||
"answer": "partial answer",
|
||||
"error": "judge failed",
|
||||
"score": None,
|
||||
"answer_result": {"answer": "different", "error": "answer failed"},
|
||||
"judge_result": {"answer": "invalid judgement", "metadata": {"reason": "diagnostic"}},
|
||||
"exception": {"type": "TimeoutError", "causes": [{"type": "ReadTimeout"}]},
|
||||
}
|
||||
assert compact_query(raw) == raw
|
||||
|
||||
|
||||
def test_construction_compaction_preserves_jobs_and_only_drops_reconstructable_data():
|
||||
raw = {
|
||||
"success": True,
|
||||
"jobs": [
|
||||
{
|
||||
"job": "auto_memory",
|
||||
"memory_checkpoint": {"commit_sha": "abc", "message": "session: s1"},
|
||||
"result": {
|
||||
"success": True,
|
||||
"job_call_counts": {"auto_memory": 2, "daily_list": 1},
|
||||
"token_usage": {
|
||||
"answer": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13},
|
||||
"empty": {"total_tokens": None},
|
||||
},
|
||||
"metadata": {
|
||||
"source_conversation": "[[session/dialog/s1.jsonl]]",
|
||||
"details": [
|
||||
{"session_id": "s1", "source_conversation": "[[session/dialog/s1.jsonl]]"},
|
||||
{"session_id": "s2", "source_conversation": "[[session/dialog/other.jsonl]]"},
|
||||
],
|
||||
"plugin_payload": {"index": {"notes": ["keep", "this"]}},
|
||||
"index": {
|
||||
"date": "2026-01-01",
|
||||
"path": "custom/daily/2026-01-01.md",
|
||||
"created": False,
|
||||
"changed": True,
|
||||
"notes": [{"text": "old"}, {"text": "new"}],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"job": "index_update",
|
||||
"result": {
|
||||
"job_call_counts": {"index_update": 1},
|
||||
"token_usage": {"answer": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}},
|
||||
},
|
||||
},
|
||||
{"job": "index_update", "result": {"token_usage": {"empty": {"total_tokens": None}}}},
|
||||
],
|
||||
}
|
||||
original = deepcopy(raw)
|
||||
result = compact_construction(raw)
|
||||
assert raw == original
|
||||
job = result["jobs"][0]
|
||||
assert "memory_checkpoint" not in job
|
||||
assert job["result"]["job_call_counts"] == {"auto_memory": 1, "daily_list": 1}
|
||||
assert set(job["result"]["token_usage"]) == {"answer"}
|
||||
metadata = job["result"]["metadata"]
|
||||
assert metadata["source_conversation"] == "[[session/dialog/s1.jsonl]]"
|
||||
assert metadata["details"] == [
|
||||
{"session_id": "s1"},
|
||||
{"session_id": "s2", "source_conversation": "[[session/dialog/other.jsonl]]"},
|
||||
]
|
||||
assert metadata["plugin_payload"] == {"index": {"notes": ["keep", "this"]}}
|
||||
assert metadata["index"] == {
|
||||
"date": "2026-01-01",
|
||||
"path": "custom/daily/2026-01-01.md",
|
||||
"created": False,
|
||||
"changed": True,
|
||||
"notes_count": 2,
|
||||
}
|
||||
assert result["jobs"][1]["result"] == {"token_usage": 0}
|
||||
assert result["jobs"][2]["result"] == {}
|
||||
|
||||
|
||||
def test_summary_matches_reviewed_dev_meta_fields_and_denominators(tmp_path):
|
||||
cases = [
|
||||
CaseSpec(
|
||||
case_id=case_id,
|
||||
sessions=[],
|
||||
queries=[QuerySpec(query_id=query_id, question="?", golden_answer="a") for query_id in ids],
|
||||
)
|
||||
for case_id, ids in [("one", ["q1", "q2"]), ("two", ["q3"])]
|
||||
]
|
||||
write_json(
|
||||
tmp_path / "cases/one/memory_construction/result.json",
|
||||
{
|
||||
"status": "partial_success",
|
||||
"jobs": [{"result": {"success": True}}, {"result": {"success": False}}],
|
||||
},
|
||||
)
|
||||
write_json(
|
||||
tmp_path / "cases/one/queries/result.json",
|
||||
{
|
||||
"queries": [
|
||||
{
|
||||
"query_id": "q1",
|
||||
"question_type": "fact",
|
||||
"score": 0.5,
|
||||
"answer_result": {"token_usage": {"answer": {"total_tokens": 12}}},
|
||||
},
|
||||
{
|
||||
"query_id": "q2",
|
||||
"question_type": "fact",
|
||||
"score": None,
|
||||
"error": "candidate bug",
|
||||
"answer_result": {},
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
write_json(tmp_path / "cases/two/memory_construction/result.json", {"status": "failure", "jobs": []})
|
||||
write_json(
|
||||
tmp_path / "cases/two/failure.json",
|
||||
{"stage": "construction", "failure_kind": "candidate_failure", "error": "build failed"},
|
||||
)
|
||||
summary = summarize(tmp_path, cases)
|
||||
assert summary == {
|
||||
"case_count": 2,
|
||||
"case_status_counts": {"completed": 0, "candidate_failure": 2, "infra_error": 0},
|
||||
"construction_status_counts": {"success": 0, "partial_success": 1, "failure": 1},
|
||||
"failure_stage_case_counts": {"construction": 2, "query": 1},
|
||||
"expected_query_count": 3,
|
||||
"scored_query_count": 1,
|
||||
"query_status_counts": {"completed": 1, "candidate_failure": 1, "infra_error": 0, "unscored": 2},
|
||||
"mean_query_score": 0.5 / 3,
|
||||
"observed_mean_query_score": 0.5,
|
||||
"low_score_cases": [{"one": 0.25}],
|
||||
"question_type_statistics": {
|
||||
"fact": {"query_count": 2, "average_score": 0.5, "average_query_total_token_usage": {"answer": 6.0}},
|
||||
},
|
||||
"cases": {
|
||||
"one": {
|
||||
"construction": {"status": "partial_success", "success_job_count": 1, "fail_job_count": 1},
|
||||
"mean_score": 0.25,
|
||||
"average_query_total_token_usage": {"answer": 6.0},
|
||||
"queries": {
|
||||
"q1": {"question_type": "fact", "score": 0.5, "status": "completed"},
|
||||
"q2": {"question_type": "fact", "score": None, "status": "candidate_failure"},
|
||||
},
|
||||
"construction_error_count": 1,
|
||||
"query_error_count": 1,
|
||||
},
|
||||
"two": {
|
||||
"construction": {"status": "failure", "success_job_count": 0, "fail_job_count": 0},
|
||||
"mean_score": 0,
|
||||
"average_query_total_token_usage": {},
|
||||
"queries": {},
|
||||
"stage": "construction",
|
||||
},
|
||||
},
|
||||
}
|
||||
write_json(
|
||||
tmp_path / "manifest.json",
|
||||
{
|
||||
"status": "completed",
|
||||
"mode": "scored",
|
||||
"case_ids": ["one", "two"],
|
||||
"query_ids": {"one": ["q1", "q2"], "two": ["q3"]},
|
||||
},
|
||||
)
|
||||
assert rebuild_summary(tmp_path) == summary
|
||||
assert read_json(tmp_path / "summary.json") == summary
|
||||
failure_path = tmp_path / "cases/two/failure.json"
|
||||
failure = read_json(failure_path)
|
||||
failure["failure_kind"] = "infra_error"
|
||||
write_json(failure_path, failure)
|
||||
infra = rebuild_summary(tmp_path)
|
||||
assert infra["mean_query_score"] is None
|
||||
assert infra["observed_mean_query_score"] == 0.5
|
||||
assert infra["cases"]["two"]["mean_score"] is None
|
||||
assert infra["cases"]["one"]["mean_score"] == 0.25
|
||||
|
||||
|
||||
def test_token_averages_exclude_zero_unknown_and_judge_cost_but_count_all_returned_queries():
|
||||
assert average_tokens(
|
||||
[
|
||||
{
|
||||
"answer_result": {"token_usage": {"answer": {"total_tokens": 12}, "zero": {"total_tokens": 0}}},
|
||||
"judge_result": {"token_usage": {"judge": {"total_tokens": 100}}},
|
||||
},
|
||||
{"answer_result": {"token_usage": {"unknown": {"total_tokens": None}, "bad": {"total_tokens": True}}}},
|
||||
{},
|
||||
],
|
||||
) == {"answer": 4.0}
|
||||
|
||||
|
||||
def test_compact_json_round_trips_strings_and_keeps_small_leaves_inline(tmp_path):
|
||||
value = {"result": {"token_usage": 0}, "unicode": ["你好", 'quote"\n'], "large": [1, 2, 3]}
|
||||
path = tmp_path / "result.json"
|
||||
write_json(path, value, compact_leaf_containers=True)
|
||||
assert read_json(path) == value
|
||||
assert '"result": {"token_usage": 0}' in path.read_text()
|
||||
assert "\\u4f60" not in path.read_text()
|
||||
with pytest.raises(ValueError):
|
||||
write_json(path, {"score": float("nan")}, compact_leaf_containers=True)
|
||||
assert json.loads(path.read_text()) == value
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stop", [RuntimeError("judge unavailable"), asyncio.CancelledError()])
|
||||
async def test_judge_interruption_preserves_all_frozen_answers_and_compacts_each_response_once(tmp_path, stop):
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from deploy.benchmark_docker.pool import CaseResult
|
||||
from reme_home.evaluation.component import HomeEvaluation
|
||||
from reme_home.evaluation.inputs import EvaluationSpec
|
||||
|
||||
case = CaseSpec(
|
||||
case_id="case",
|
||||
sessions=[],
|
||||
queries=[QuerySpec(query_id=f"q{i}", question="?", golden_answer="a") for i in range(2)],
|
||||
)
|
||||
answers = [
|
||||
{
|
||||
"query_id": query.query_id,
|
||||
"answer": "frozen answer",
|
||||
"error": None,
|
||||
"answer_result": {"answer": "frozen answer", "job_call_counts": {"agentic_answer": 2}},
|
||||
}
|
||||
for query in case.queries
|
||||
]
|
||||
original = deepcopy(answers)
|
||||
result = CaseResult("case", build={"success": True, "jobs": []}, queries={"queries": answers})
|
||||
spec = EvaluationSpec(fail_fast=True)
|
||||
await HomeEvaluation._persist_case(tmp_path, case, result, None, spec) # pylint: disable=protected-access
|
||||
judge = SimpleNamespace(evaluate=AsyncMock(side_effect=stop))
|
||||
with pytest.raises(type(stop)):
|
||||
await HomeEvaluation._score_case( # pylint: disable=protected-access
|
||||
tmp_path,
|
||||
case,
|
||||
"beam",
|
||||
judge,
|
||||
asyncio.Semaphore(1),
|
||||
spec,
|
||||
answers,
|
||||
)
|
||||
rows = read_json(tmp_path / "cases/case/queries/result.json")["queries"]
|
||||
assert [row["answer"] for row in rows] == ["frozen answer", "frozen answer"]
|
||||
assert [row["answer_result"]["job_call_counts"] for row in rows] == [{"agentic_answer": 1}] * 2
|
||||
assert answers == original
|
||||
assert not (tmp_path / "cases/case/queries/answers.json").exists()
|
||||
|
||||
|
||||
def test_rebuild_rejects_missing_completed_answers_without_replacing_summary(tmp_path):
|
||||
write_json(
|
||||
tmp_path / "manifest.json",
|
||||
{"status": "completed", "case_ids": ["case"], "query_ids": {"case": ["q"]}},
|
||||
)
|
||||
write_json(tmp_path / "cases/case/memory_construction/result.json", {"status": "success", "jobs": []})
|
||||
write_json(tmp_path / "summary.json", {"preserve": "existing summary"})
|
||||
with pytest.raises(ValueError, match="incomplete query results"):
|
||||
rebuild_summary(tmp_path)
|
||||
assert read_json(tmp_path / "summary.json") == {"preserve": "existing summary"}
|
||||
Loading…
Add table
Reference in a new issue