feat(home): add standalone benchmark test evaluation and fix compression recovery

This commit is contained in:
酱牛肉 2026-11-02 14:44:37 +08:00
parent abf36eceec
commit 0e4ec5d287
15 changed files with 854 additions and 15 deletions

View file

@ -192,3 +192,39 @@ from benchmark.beam.run_home import run
result = await run(workspace_dir="/path/to/new-workspace")
```
## Evaluate a candidate on an independent test set
The default evaluates every 100K case, matching the previous standalone evaluation. This is an independent
workspace, not a disjoint held-out split. Select another `variant`, or replace `all_cases: true` with
`case_ids: [...]`, in [config_home_eval.yaml](config_home_eval.yaml) for a different test set.
The entry point imports a full candidate SHA or unambiguous hexadecimal prefix from an existing Home
optimization workspace and copies its prepared fixed runtime. It does not require the original ReMe source
repository or change the optimizer's checkout. Paths in the configuration are repository-relative.
Prepare first, then use the service from two terminals (replace `CANDIDATE_SHA`):
```bash
python benchmark/beam/run_home_eval.py --commit CANDIDATE_SHA --prepare-only
# Terminal 1:
reme start workspace_dir=benchmark/beam/workspaces/home-test-100K 'plugins=["home","beam-judge"]'
# Terminal 2:
reme home_evaluate workspace_dir=benchmark/beam/workspaces/home-test-100K record_history=false
```
### Local one-shot test evaluation
```bash
python benchmark/beam/run_home_eval.py --commit CANDIDATE_SHA
# Optional overrides: --config /path/to/test.yaml --source-workspace /path/to/search --test-workspace /path/to/test
```
Python callers can use `await benchmark.beam.run_home_eval.run(commit="CANDIDATE_SHA")`.
The runner rebuilds construction and scores every selected query with sampling and history recording disabled.
It uses the existing Docker evaluation runtime and host judge configuration; model credentials are required
for evaluation. Results stay in the test workspace's `home/evaluations/`, with original candidate provenance
in each manifest's `test_evaluation`. The exported snapshot has its own local `evaluation_commit_sha`.
Rerunning the same request reopens unchanged prepared inputs and creates another evaluation result. Use a
new test workspace for another candidate or selection. Source and test workspaces must not overlap.

View file

@ -172,3 +172,38 @@ python benchmark/beam/run_home.py --optimize
不加 `--optimize` 时仍只准备。
默认选择 BEAM **100K 全部 case**。
## 独立测试集评测
默认在独立工作区评测 **100K 全部 case**,与旧独立评测入口一致,不代表与训练集互斥的留出集。
可在 [config_home_eval.yaml](config_home_eval.yaml) 更换 `variant`,或把 `all_cases: true`
替换为 `case_ids: [...]`,指定测试数据。
入口从已有 Home 优化工作区导出指定候选提交,接受完整 SHA 或无歧义的十六进制短 SHA,
并复制该实验已准备的固定 runtime。无需原始 ReMe 源码仓库,也不会切换或修改优化工作区。
配置内路径均相对仓库根目录。
先准备测试工作区,再分别在两个终端启动服务和调用评测(替换 `CANDIDATE_SHA`):
```bash
python benchmark/beam/run_home_eval.py --commit CANDIDATE_SHA --prepare-only
# 终端 1:
reme start workspace_dir=benchmark/beam/workspaces/home-test-100K 'plugins=["home","beam-judge"]'
# 终端 2:
reme home_evaluate workspace_dir=benchmark/beam/workspaces/home-test-100K record_history=false
```
### 本地一次性测试评测
```bash
python benchmark/beam/run_home_eval.py --commit CANDIDATE_SHA
# 可选:--config /path/to/test.yaml --source-workspace /path/to/search --test-workspace /path/to/test
```
Python 可调用 `await benchmark.beam.run_home_eval.run(commit="CANDIDATE_SHA")`。
入口完整重建记忆并对所选 case 的全部 query 评分,关闭采样与优化历史记录,使用现有 Docker
评测环境和宿主 judge 配置;实际评测需要模型凭证。结果保存在测试工作区的 `home/evaluations/`,
manifest 的 `test_evaluation` 记录来源候选,导出后的本地提交另记为 `evaluation_commit_sha`。
相同请求可复用未改动的准备结果并新增一次评测;更换候选或数据选择时需使用新测试工作区。
测试工作区与来源工作区必须互不包含。

View file

@ -0,0 +1,13 @@
# Independent full-dataset evaluation; paths are relative to the ReMe repository.
source_workspace: benchmark/beam/workspaces/home-100K
test_workspace: benchmark/beam/workspaces/home-test-100K
code_commit: "" # Required: set a candidate SHA/prefix, or pass --commit.
benchmark:
name: beam
source: benchmark/beam/dataset/BEAM
variant: 100K
all_cases: true # Full 100K evaluation; not a disjoint held-out split.
evaluation:
concurrency: 4
timeout_seconds: 7200
record_history: false

View file

@ -0,0 +1,21 @@
"""Evaluate a BEAM Home candidate in an independent test workspace."""
import sys
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[2]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from benchmark.home_eval import main, run as run_evaluation # noqa: E402 # pylint: disable=wrong-import-position
DEFAULT_CONFIG = Path(__file__).with_name("config_home_eval.yaml")
async def run(config_path=DEFAULT_CONFIG, **kwargs):
"""Score the selected BEAM commit without changing its optimization workspace."""
return await run_evaluation(config_path, expected_benchmark="beam", **kwargs)
if __name__ == "__main__":
main("beam", DEFAULT_CONFIG)

207
benchmark/home_eval.py Normal file
View file

@ -0,0 +1,207 @@
"""Standalone test evaluation shared by the BEAM and LongMemEval entry points."""
import argparse
import asyncio
import json
from pathlib import Path
from pydantic import Field, model_validator
import yaml
from reme_home.code.repository import resolve_revision, validate_repository
from reme_home.evaluation.artifacts import contained, read_json
from reme_home.evaluation.component import blocking
from reme_home.evaluation.inputs import EvaluationSpec, load_inputs
from reme_home.judge import JUDGE_PLUGINS
from reme_home.preparation.code import tree_fingerprint
from reme_home.preparation.data.models import fingerprint
from reme_home.preparation.evaluation_source import EvaluationSource
from reme_home.preparation.spec import BenchmarkSpec, DebugSpec, SpecModel
from reme.application import Application
PROJECT_ROOT = Path(__file__).resolve().parents[1]
class TestRunConfig(SpecModel):
"""Explicit dataset selection and execution options for one pinned candidate."""
source_workspace: str = Field(min_length=1)
test_workspace: str = Field(min_length=1)
code_commit: str = Field(min_length=4)
benchmark: BenchmarkSpec
exclude_training_cases: bool = False
evaluation: EvaluationSpec = Field(default_factory=lambda: EvaluationSpec(record_history=False))
@model_validator(mode="after")
def validate_test_mode(self):
"""Test runs score every prepared query and never consume optimizer budgets."""
options = self.evaluation
if any(
(
options.debug,
options.sampling,
options.record_history,
options.max_validations is not None,
options.sampling_policy is not None,
options.case_ids is not None,
options.query_ids is not None,
options.reuse_construction_from is not None,
),
):
raise ValueError(
"Test evaluation requires full scoring with record_history=false; select cases in benchmark",
)
if self.exclude_training_cases and (self.benchmark.name != "longmemeval" or not self.benchmark.all_cases):
raise ValueError("exclude_training_cases requires LongMemEval with all_cases=true")
return self
def project_path(value: str, project_root: Path) -> Path:
"""Resolve runner paths independently of the launching working directory."""
if value.startswith("~"):
raise ValueError("Use absolute or project-relative paths, not home-relative paths")
return (project_root / value).resolve()
def load_request(config_path, *, commit=None, source_workspace=None, test_workspace=None, project_root=PROJECT_ROOT):
"""Validate all source inputs before starting an application or writing test data."""
raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
for key, value in (
("code_commit", commit),
("source_workspace", source_workspace),
("test_workspace", test_workspace),
):
if value is not None:
raw[key] = value
# Execution settings can be partial while retaining the test-only history default.
raw["evaluation"] = {"record_history": False, **raw.get("evaluation", {})}
config = TestRunConfig.model_validate(raw)
source_path = project_path(config.source_workspace, project_root)
target = project_path(config.test_workspace, project_root)
if target.is_relative_to(source_path) or source_path.is_relative_to(target):
raise ValueError("Test and source workspaces must be separate, non-nested directories")
if target.exists() and (
not target.is_dir() or (any(target.iterdir()) and not (target / "home/metadata/workspace.json").is_file())
):
raise ValueError("Use an empty test workspace or a compatible prepared Home test workspace")
source = EvaluationSource.load(source_path, config.code_commit)
benchmark = config.benchmark.model_copy(deep=True)
if benchmark.name != source.spec.benchmark.name:
raise ValueError("Test benchmark must match the source experiment")
benchmark.source = str(project_path(benchmark.source, project_root).resolve(strict=True))
if benchmark.evidence_source is not None:
benchmark.evidence_source = str(project_path(benchmark.evidence_source, project_root).resolve(strict=True))
if config.exclude_training_cases:
items = json.loads(Path(benchmark.source).read_text(encoding="utf-8"))
ids = [str(item["question_id"]) for item in items]
if len(ids) != len(set(ids)):
raise ValueError("Duplicate LongMemEval question IDs")
training = set(source.record["dataset"]["case_ids"])
if training - set(ids):
raise ValueError("Test dataset is missing training IDs; provide the original full dataset")
benchmark.case_ids = [case_id for case_id in ids if case_id not in training]
benchmark.all_cases = False
benchmark = BenchmarkSpec.model_validate(benchmark.model_dump())
spec = source.spec.model_copy(deep=True)
spec.benchmark = benchmark
spec.debug = DebugSpec(enabled=False)
return config, target, source, spec
def existing_preparation(home, source, spec):
"""Reopen only the same request with unchanged code, runtime and normalized data."""
marker = contained(home, "metadata/preparation.json")
if not marker.exists():
return None
record = read_json(marker)
expected = {**source.provenance, "request_fingerprint": fingerprint(spec.model_dump(mode="json"))}
if record.get("test_evaluation") != expected:
raise ValueError("Test workspace belongs to a different candidate or dataset; use a new workspace")
domain = yaml.safe_load(contained(home, "domain_spec.yaml").read_text(encoding="utf-8"))
if fingerprint(domain) != record["config_fingerprint"]:
raise ValueError("Test preparation configuration has changed")
repository, _ = validate_repository(home, record)
if (
resolve_revision(repository, "HEAD") != record["code"]["initial_commit"]
or tree_fingerprint(repository) != record["code"]["candidate_fingerprint"]
or tree_fingerprint(contained(home, "code/runtime", directory=True)) != source.provenance["runtime_fingerprint"]
):
raise ValueError("Prepared test code or runtime has changed")
load_inputs(home, EvaluationSpec(record_history=False))
return record
async def run(
config_path,
*,
commit=None,
source_workspace=None,
test_workspace=None,
prepare_only=False,
expected_benchmark=None,
):
"""Prepare transactionally, then score through the normal application lifecycle."""
config, target, source, spec = await blocking(
load_request,
config_path,
commit=commit,
source_workspace=source_workspace,
test_workspace=test_workspace,
)
if expected_benchmark is not None and config.benchmark.name != expected_benchmark:
raise ValueError(f"This entry point requires benchmark.name={expected_benchmark}")
app = Application(
workspace_dir=str(target),
plugins=["home", JUDGE_PLUGINS[config.benchmark.name]],
service={"backend": "cli"},
enable_logo=False,
log_to_file=False,
)
try:
await app.start()
home = target / "home"
prepared = await blocking(existing_preparation, home, source, spec)
if prepared is None:
preparation = app.context.components["home.preparation"]["default"]
prepared = await preparation.prepare(spec=spec.model_dump(mode="json"), code_source=source)
result = {
"test_workspace": str(target),
**source.provenance,
"evaluation_commit_sha": prepared["code"]["initial_commit"],
"case_count": prepared["dataset"]["case_count"],
"query_count": prepared["dataset"]["query_count"],
}
if not prepare_only:
response = await app.run_job(
"home_evaluate",
**config.evaluation.model_dump(mode="json", exclude_unset=True),
)
if not response.success:
raise RuntimeError(response.answer)
result["evaluation"] = response.answer
return result
finally:
await app.close()
def main(benchmark, default_config):
"""Explicit local one-shot entry point, with preparation-only service setup."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", type=Path, default=default_config)
parser.add_argument("--commit", help="Candidate commit SHA or unambiguous hexadecimal prefix")
parser.add_argument("--source-workspace", help="Prepared optimization workspace")
parser.add_argument("--test-workspace", help="Independent test output workspace")
parser.add_argument("--prepare-only", action="store_true", help="Prepare for a later home_evaluate service call")
args = parser.parse_args()
result = asyncio.run(
run(
args.config,
commit=args.commit,
source_workspace=args.source_workspace,
test_workspace=args.test_workspace,
prepare_only=args.prepare_only,
expected_benchmark=benchmark,
),
)
print(json.dumps(result, ensure_ascii=False, indent=2))

View file

@ -174,3 +174,40 @@ out. The runner reads the IDs directly from YAML without loading group files.
[home_evidence_sessions.jsonl](home_evidence_sessions.jsonl) is a byte-for-byte
copy of that revision's `dataset/longmemeval_evidence_sessions.jsonl`;
reviewed evidence-session annotations are preserved during normalization.
## Evaluate a candidate on an independent test set
The default excludes the source experiment's actual training IDs from the full dataset. With the default
250-case training workspace this selects held-out groups 6–10. For an explicit selection, set
`exclude_training_cases: false` and replace `all_cases: true` with `case_ids: [...]` in
[config_home_eval.yaml](config_home_eval.yaml). An empty held-out selection fails before preparation.
The entry point imports a full candidate SHA or unambiguous hexadecimal prefix from an existing Home
optimization workspace and copies its prepared fixed runtime. It does not require the original ReMe source
repository or change the optimizer's checkout. Paths in the configuration are repository-relative.
Prepare first, then use the service from two terminals (replace `CANDIDATE_SHA`):
```bash
python benchmark/longmemeval/run_home_eval.py --commit CANDIDATE_SHA --prepare-only
# Terminal 1:
reme start workspace_dir=benchmark/longmemeval/workspaces/home-test-held-out 'plugins=["home","lme-judge"]'
# Terminal 2:
reme home_evaluate workspace_dir=benchmark/longmemeval/workspaces/home-test-held-out record_history=false
```
### Local one-shot test evaluation
```bash
python benchmark/longmemeval/run_home_eval.py --commit CANDIDATE_SHA
# Optional overrides: --config /path/to/test.yaml --source-workspace /path/to/search --test-workspace /path/to/test
```
Python callers can use `await benchmark.longmemeval.run_home_eval.run(commit="CANDIDATE_SHA")`.
The runner rebuilds construction and scores every selected query with sampling and history recording disabled.
It uses the existing Docker evaluation runtime and host judge configuration; model credentials are required
for evaluation. Results stay in the test workspace's `home/evaluations/`, with original candidate provenance
in each manifest's `test_evaluation`. The exported snapshot has its own local `evaluation_commit_sha`.
Rerunning the same request reopens unchanged prepared inputs and creates another evaluation result. Use a
new test workspace for another candidate or selection. Source and test workspaces must not overlap.

View file

@ -147,3 +147,39 @@ ID 已按原顺序直接写入 [config_home.yaml](config_home.yaml) 的 `benchma
运行时无需读取分组文件,group_6–group_10 保持为留出集。
独立审阅的证据标注保存在
[home_evidence_sessions.jsonl](home_evidence_sessions.jsonl),来源见英文说明。
## 独立测试集评测
默认从完整数据中排除来源工作区实际使用的训练 ID;使用默认 250 个训练 case 时,测试集为
**group_6–group_10,共 250 个 case**。若要显式选取测试 case,可在
[config_home_eval.yaml](config_home_eval.yaml) 设置 `exclude_training_cases: false`,
并把 `all_cases: true` 替换为 `case_ids: [...]`。留出集为空时会直接报错。
入口从已有 Home 优化工作区导出指定候选提交,接受完整 SHA 或无歧义的十六进制短 SHA,
并复制该实验已准备的固定 runtime。无需原始 ReMe 源码仓库,也不会切换或修改优化工作区。
配置内路径均相对仓库根目录。
先准备测试工作区,再分别在两个终端启动服务和调用评测(替换 `CANDIDATE_SHA`):
```bash
python benchmark/longmemeval/run_home_eval.py --commit CANDIDATE_SHA --prepare-only
# 终端 1:
reme start workspace_dir=benchmark/longmemeval/workspaces/home-test-held-out 'plugins=["home","lme-judge"]'
# 终端 2:
reme home_evaluate workspace_dir=benchmark/longmemeval/workspaces/home-test-held-out record_history=false
```
### 本地一次性测试评测
```bash
python benchmark/longmemeval/run_home_eval.py --commit CANDIDATE_SHA
# 可选:--config /path/to/test.yaml --source-workspace /path/to/search --test-workspace /path/to/test
```
Python 可调用 `await benchmark.longmemeval.run_home_eval.run(commit="CANDIDATE_SHA")`。
入口完整重建记忆并对所选 case 的全部 query 评分,关闭采样与优化历史记录,使用现有 Docker
评测环境和宿主 judge 配置;实际评测需要模型凭证。结果保存在测试工作区的 `home/evaluations/`,
manifest 的 `test_evaluation` 记录来源候选,导出后的本地提交另记为 `evaluation_commit_sha`。
相同请求可复用未改动的准备结果并新增一次评测;更换候选或数据选择时需使用新测试工作区。
测试工作区与来源工作区必须互不包含。

View file

@ -0,0 +1,14 @@
# Evaluate the complement of the source experiment's training IDs in the full dataset.
# With the default training workspace this selects held-out groups 6-10 (250 cases).
source_workspace: benchmark/longmemeval/workspaces/home-groups-1-5
test_workspace: benchmark/longmemeval/workspaces/home-test-held-out
code_commit: "" # Required: set a candidate SHA/prefix, or pass --commit.
exclude_training_cases: true
benchmark:
name: longmemeval
source: benchmark/longmemeval/dataset/longmemeval_s_reme_cleaned.json
all_cases: true
evaluation:
concurrency: 4
timeout_seconds: 7200
record_history: false

View file

@ -0,0 +1,21 @@
"""Evaluate a LongMemEval Home candidate on an independent test dataset."""
import sys
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[2]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from benchmark.home_eval import main, run as run_evaluation # noqa: E402 # pylint: disable=wrong-import-position
DEFAULT_CONFIG = Path(__file__).with_name("config_home_eval.yaml")
async def run(config_path=DEFAULT_CONFIG, **kwargs):
"""Score the selected LongMemEval commit in its own Home workspace."""
return await run_evaluation(config_path, expected_benchmark="longmemeval", **kwargs)
if __name__ == "__main__":
main("longmemeval", DEFAULT_CONFIG)

View file

@ -5,7 +5,7 @@ import logging
from pathlib import Path
from uuid import uuid4
from agentscope.exception import ToolJSONDecodeError
from agentscope.exception import StructuredOutputError, ToolJSONDecodeError
from agentscope.message import TextBlock
import jsonschema
import openai
@ -67,7 +67,16 @@ class RecoverableCompressionAgent(BoundedReActAgent):
"""Match dev_meta's model/output failures, excluding uncompressible prompts."""
if isinstance(exc, RuntimeError):
return not str(exc).startswith("The system prompt")
return isinstance(exc, (ToolJSONDecodeError, jsonschema.ValidationError, ValidationError, openai.OpenAIError))
return isinstance(
exc,
(
StructuredOutputError,
ToolJSONDecodeError,
jsonschema.ValidationError,
ValidationError,
openai.OpenAIError,
),
)
async def _fallback_compress_context(self, cause, context_ratio, config):
"""Persist older messages before replacing state; never call a second model."""

View file

@ -291,6 +291,8 @@ class HomeEvaluation(BaseComponent):
}
if source is not None:
manifest["construction_source"] = source
if "test_evaluation" in record:
manifest["test_evaluation"] = record["test_evaluation"]
if admission is not None:
manifest["candidate_gate"] = admission
if reuse is not None:

View file

@ -14,6 +14,7 @@ from reme.components.base_component import BaseComponent
from reme.steps.base_step import BaseStep, Ref
from .code import prepare_code
from .evaluation_source import EvaluationSource
from .data import prepare_dataset
from .data.basic import derive_debug_cases
from .data.models import fingerprint
@ -58,10 +59,18 @@ class HomePreparation(BaseComponent):
if self._pending:
await asyncio.gather(*self._pending, return_exceptions=True)
async def prepare(self, *, spec: dict | None = None, spec_path: str | None = None) -> dict:
async def prepare(
self,
*,
spec: dict | None = None,
spec_path: str | None = None,
code_source: EvaluationSource | None = None,
) -> dict:
"""Keep blocking data/Git work off the event loop and await rollback on cancellation."""
stop = Event()
task = asyncio.create_task(asyncio.to_thread(self._prepare, spec=spec, spec_path=spec_path, stop=stop))
task = asyncio.create_task(
asyncio.to_thread(self._prepare, spec=spec, spec_path=spec_path, stop=stop, code_source=code_source),
)
self._pending.add(task)
cancelled = False
try:
@ -80,7 +89,7 @@ class HomePreparation(BaseComponent):
finally:
self._pending.discard(task)
def _prepare(self, *, spec: dict | None, spec_path: str | None, stop: Event) -> dict:
def _prepare(self, *, spec: dict | None, spec_path: str | None, stop: Event, code_source=None) -> dict:
"""Prepare a started application's workspace from exactly one input form.
Local YAML paths must stay inside the workspace. Explicit dataset/repository
@ -108,19 +117,23 @@ class HomePreparation(BaseComponent):
self.judge.require_benchmark(config.benchmark.name)
# Resolve explicitly supplied source inputs before creating transaction state.
source = _source_path(workspace, config.benchmark.source)
repository = _source_path(workspace, config.runtime.source_repo)
repository = _source_path(workspace, config.runtime.source_repo) if code_source is None else None
evidence = (
_source_path(workspace, config.benchmark.evidence_source) if config.benchmark.evidence_source else None
)
config.benchmark.source = str(source)
config.runtime.source_repo = str(repository)
if repository is not None:
config.runtime.source_repo = str(repository)
config.benchmark.evidence_source = str(evidence) if evidence else None
existed = check_targets(root, state)
transaction = state / TRANSACTION
transaction.mkdir()
try:
sync_directory(state)
result = self._stage(config, transaction, source, repository, evidence)
if code_source is None:
result = self._stage(config, transaction, source, repository, evidence)
else:
result = self._stage(config, transaction, source, repository, evidence, code_source=code_source)
def check_cancelled():
if stop.is_set():
@ -142,7 +155,8 @@ class HomePreparation(BaseComponent):
return result
@staticmethod
def _stage(config, transaction, source, repository, evidence) -> dict:
def _stage(config, transaction, source, repository, evidence, *, code_source=None) -> dict:
request_fingerprint = fingerprint(config.model_dump(mode="json"))
dataset = prepare_dataset(
transaction / "output-2",
config.benchmark.name,
@ -163,7 +177,11 @@ class HomePreparation(BaseComponent):
)
else:
(transaction / "output-3/cases").mkdir(parents=True)
code = prepare_code(config, repository, transaction / "output-0", transaction / "output-1")
code = (
prepare_code(config, repository, transaction / "output-0", transaction / "output-1")
if code_source is None
else code_source.install(transaction / "output-0", transaction / "output-1")
)
config.runtime.source_ref = code["source_commit"]
domain = config.model_dump(mode="json")
(transaction / "output-4").write_text(yaml.safe_dump(domain, sort_keys=False), encoding="utf-8")
@ -179,6 +197,11 @@ class HomePreparation(BaseComponent):
"dataset": dataset.model_dump(mode="json"),
"code": code,
}
if code_source is not None:
result["test_evaluation"] = {
**code_source.provenance,
"request_fingerprint": request_fingerprint,
}
write_json(transaction / "output-6", result)
return result

View file

@ -0,0 +1,87 @@
"""Read-only import of a pinned Home candidate and its fixed runtime."""
from dataclasses import dataclass
from pathlib import Path
import re
import shutil
import yaml
from ..code.repository import resolve_revision, validate_repository, validate_tree
from ..code.snapshot import export_blob
from ..evaluation.artifacts import contained, read_json
from .code import git, tree_fingerprint
from .data.models import fingerprint
from .spec import PreparationSpec, relative_path
@dataclass(frozen=True)
class EvaluationSource:
"""Host-owned source identity; imports never change the source checkout."""
home: Path
commit: str
spec: PreparationSpec
record: dict
@classmethod
def load(cls, workspace: Path, commit: str):
"""Require a prepared workspace and an unambiguous commit SHA or prefix."""
if re.fullmatch(r"[0-9a-fA-F]{4,64}", commit) is None:
raise ValueError("code_commit must be a full commit SHA or hexadecimal prefix")
home = contained(workspace, "home", directory=True)
record = read_json(contained(home, "metadata/preparation.json"))
domain = yaml.safe_load(contained(home, "domain_spec.yaml").read_text(encoding="utf-8"))
if fingerprint(domain) != record["config_fingerprint"]:
raise ValueError("Source preparation configuration has changed")
spec = PreparationSpec.model_validate(domain)
repository, _ = validate_repository(home, record)
revision = resolve_revision(repository, commit)
git(repository, "merge-base", "--is-ancestor", record["code"]["initial_commit"], revision)
validate_tree(repository, record, revision)
runtime = contained(home, "code/runtime", directory=True)
if tree_fingerprint(runtime) != record["code"]["runtime_fingerprint"]:
raise ValueError("Source fixed runtime has changed since preparation")
return cls(home, revision, spec, record)
@property
def provenance(self):
"""Stable identity checked when reopening a standalone test workspace."""
return {
"source_workspace": str(self.home.parent),
"source_preparation_id": self.record["preparation_id"],
"source_commit_sha": self.commit,
"runtime_fingerprint": self.record["code"]["runtime_fingerprint"],
}
def install(self, runtime: Path, candidate: Path) -> dict:
"""Copy fixed sources and export regular Git blobs, preserving executable bits."""
source = contained(self.home, "code/runtime", directory=True)
expected = self.record["code"]["runtime_fingerprint"]
if tree_fingerprint(source) != expected:
raise ValueError("Source fixed runtime changed before import")
shutil.copytree(source, runtime)
if tree_fingerprint(runtime) != expected or tree_fingerprint(source) != expected:
raise ValueError("Source fixed runtime changed during import")
repository = contained(self.home, "code/repo", directory=True)
candidate.mkdir()
entries = git(repository, "ls-tree", "-r", "-z", "--full-tree", self.commit, strip=False)
for entry in filter(None, entries.split("\0")):
header, name = entry.split("\t", 1)
mode, kind, oid = header.split()
path = Path(relative_path(name))
if mode not in {"100644", "100755"} or kind != "blob" or ".git" in path.parts:
raise ValueError(f"Candidate must contain regular files only: {name}")
destination = candidate / path
destination.parent.mkdir(parents=True, exist_ok=True)
export_blob(repository, oid, destination)
destination.chmod(0o755 if mode == "100755" else 0o644)
# The isolated repository's init is the selected snapshot, not the optimizer baseline.
git(candidate, "init", "--initial-branch=init", "--template=")
git(candidate, "add", "--force", "--all")
git(candidate, "-c", "user.name=Home", "-c", "user.email=home@localhost", "commit", "-m", "Test candidate")
return {
**self.record["code"],
"initial_commit": resolve_revision(candidate, "HEAD"),
"candidate_fingerprint": tree_fingerprint(candidate),
}

View file

@ -0,0 +1,283 @@
"""Standalone test workspaces use pinned sources and isolated evaluation lifecycles."""
# pylint: disable=missing-function-docstring,unused-argument,protected-access,redefined-outer-name
import copy
from importlib import import_module
import json
from pathlib import Path
from subprocess import CalledProcessError
from unittest.mock import AsyncMock
import pytest
import yaml
from reme_home.evaluation.artifacts import read_json
from reme_home.evaluation.inputs import EvaluationSpec, load_inputs
from reme_home.preparation.code import tree_fingerprint
from reme_home.preparation.evaluation_source import EvaluationSource
from reme_home.evaluation import component as evaluation
from test_evaluation import FakeFactory
from test_preparation import ( # noqa: F401 # pylint: disable=unused-import
command,
home_application,
preparation_inputs,
snapshot,
)
from benchmark import home_eval
from reme.schema import Response
from reme.components.agent_wrapper import AsAgentWrapper
def write_config(tmp_path, app, spec, commit):
config = {
"source_workspace": str(app.config.workspace_dir),
"test_workspace": str(tmp_path / "test-workspace"),
"code_commit": commit,
"benchmark": copy.deepcopy(spec["benchmark"]),
"evaluation": {"concurrency": 2, "timeout_seconds": 60},
}
path = tmp_path / "test.yaml"
path.write_text(yaml.safe_dump(config))
return path, config
async def prepare_source(app, inputs):
spec, _ = inputs
response = await app.run_job("home_prepare", spec=spec)
assert response.success, response.answer
home = Path(app.config.workspace_dir) / "home"
return home, response.answer["code"]["initial_commit"]
@pytest.mark.asyncio
async def test_import_historical_commit_ignores_dirty_checkout_and_missing_original_repo(app, inputs, tmp_path):
home, _ = await prepare_source(app, inputs)
repo = home / "code/repo"
command(repo, "switch", "-c", "candidate")
file = repo / "plugins/beam/src/reme_beam/__init__.py"
file.write_text("value = 2\n")
file.chmod(0o755)
command(repo, "add", ".")
command(repo, "-c", "user.name=Test", "-c", "user.email=test@localhost", "commit", "-m", "candidate")
selected = command(repo, "rev-parse", "HEAD")
file.write_text("value = 3\n")
(repo / "untracked.txt").write_text("must not export")
spec, _ = inputs
original = Path(spec["runtime"]["source_repo"])
original.rename(tmp_path / "original-unavailable")
path, _ = write_config(tmp_path, app, spec, selected[:10])
before = snapshot(home)
result = await home_eval.run(path, prepare_only=True)
assert result["source_commit_sha"] == selected
target = Path(result["test_workspace"]) / "home"
imported = target / "code/repo/plugins/beam/src/reme_beam/__init__.py"
assert imported.read_text() == "value = 2\n"
assert imported.stat().st_mode & 0o111
assert not (target / "code/repo/untracked.txt").exists()
assert command(target / "code/repo", "branch", "--show-current") == "init"
assert tree_fingerprint(target / "code/runtime") == tree_fingerprint(home / "code/runtime")
assert read_json(target / "metadata/preparation.json")["test_evaluation"]["source_commit_sha"] == selected
assert not (target / "datasets/debug-cases/manifest.json").exists()
assert snapshot(home) == before
@pytest.mark.asyncio
async def test_reopen_scores_all_queries_without_optimizer_history_and_closes(app, inputs, tmp_path, monkeypatch):
home, commit = await prepare_source(app, inputs)
path, _ = write_config(tmp_path, app, inputs[0], commit)
await home_eval.run(path, prepare_only=True)
applications = []
factory = home_eval.Application
def tracked_application(**kwargs):
application = factory(**kwargs)
application.close = AsyncMock(wraps=application.close)
async def evaluate(name, **options):
assert name == "home_evaluate"
assert options["record_history"] is False
assert options["concurrency"] == 2
assert not options.get("sampling") and not options.get("debug")
record, _, cases = load_inputs(Path(kwargs["workspace_dir"]) / "home", EvaluationSpec(**options))
assert len(cases) == record["dataset"]["case_count"] == 2
assert sum(len(case.queries) for case in cases) == 2
return Response(answer={"result_dir": "test-results"})
application.run_job = AsyncMock(side_effect=evaluate)
applications.append(application)
return application
monkeypatch.setattr(home_eval, "Application", tracked_application)
result = await home_eval.run(path)
assert result["evaluation"] == {"result_dir": "test-results"}
applications[-1].close.assert_awaited_once()
assert not list((home / "evaluations").rglob("manifest.json"))
assert not list((home / "sampling").rglob("*.json"))
@pytest.mark.asyncio
@pytest.mark.parametrize("app", ["beam-judge", "lme-judge"], indirect=True)
async def test_full_evaluation_records_source_provenance_with_fake_docker(app, inputs, tmp_path, monkeypatch):
spec, lme = inputs
if "lme-judge" in app.config.plugins:
spec["benchmark"] = {"name": "longmemeval", "source": str(lme), "all_cases": True}
else:
for file in Path(spec["benchmark"]["source"]).glob("chats/*/*/probing_questions/probing_questions.json"):
data = read_json(file)
data["fact"][0]["rubric"] = ["says hello"]
file.write_text(json.dumps(data))
home, commit = await prepare_source(app, inputs)
path, _ = write_config(tmp_path, app, spec, commit)
monkeypatch.setattr(evaluation, "DockerReMeSandboxFactory", FakeFactory)
answer = "yes" if spec["benchmark"]["name"] == "longmemeval" else '{"score": 0.5}'
monkeypatch.setattr(AsAgentWrapper, "reply", AsyncMock(return_value={"result": answer}))
before = snapshot(home)
result = await home_eval.run(path)
run = Path(result["evaluation"]["result_dir"])
manifest = read_json(run / "manifest.json")
assert manifest["test_evaluation"]["source_commit_sha"] == commit
assert manifest["commit_sha"] == result["evaluation_commit_sha"]
assert manifest["execution_mode"] == "full_validation"
assert manifest["options"]["record_history"] is False
assert result["evaluation"]["scored_query_count"] == result["query_count"]
assert result["evaluation"]["mean_query_score"] == (1.0 if answer == "yes" else 0.5)
assert snapshot(home) == before
@pytest.mark.asyncio
async def test_evaluation_failure_closes_application_and_preserves_preparation(app, inputs, tmp_path, monkeypatch):
_, commit = await prepare_source(app, inputs)
path, config = write_config(tmp_path, app, inputs[0], commit)
factory = home_eval.Application
applications = []
def failing_application(**kwargs):
application = factory(**kwargs)
application.run_job = AsyncMock(return_value=Response(success=False, answer="Infrastructure failure"))
application.close = AsyncMock(wraps=application.close)
applications.append(application)
return application
monkeypatch.setattr(home_eval, "Application", failing_application)
with pytest.raises(RuntimeError, match="Infrastructure failure"):
await home_eval.run(path)
applications[-1].close.assert_awaited_once()
assert (Path(config["test_workspace"]) / "home/metadata/preparation.json").is_file()
@pytest.mark.asyncio
@pytest.mark.parametrize("app", ["lme-judge"], indirect=True)
async def test_lme_default_selection_excludes_actual_training_cases(app, inputs, tmp_path):
spec, lme = inputs
original = json.loads(lme.read_text())
held_out = {**original[0], "question_id": "q2"}
lme.write_text(json.dumps([*original, held_out]))
spec["benchmark"] = {"name": "longmemeval", "source": str(lme), "case_ids": ["q1"]}
_, commit = await prepare_source(app, inputs)
path, config = write_config(tmp_path, app, spec, commit)
config["benchmark"] = {"name": "longmemeval", "source": str(lme), "all_cases": True}
config["exclude_training_cases"] = True
path.write_text(yaml.safe_dump(config))
result = await home_eval.run(path, prepare_only=True, expected_benchmark="longmemeval")
target = Path(result["test_workspace"]) / "home"
_, dataset, cases = load_inputs(target, EvaluationSpec(record_history=False))
assert dataset.case_ids == ["q2"]
assert [case.case_id for case in cases] == ["q2"]
assert result["case_count"] == result["query_count"] == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("change", ["candidate", "dataset", "runtime", "test_code", "test_data"])
async def test_reopen_rejects_changed_identity_or_artifacts(app, inputs, tmp_path, change):
_, commit = await prepare_source(app, inputs)
path, config = write_config(tmp_path, app, inputs[0], commit)
result = await home_eval.run(path, prepare_only=True)
home = Path(result["test_workspace"]) / "home"
if change == "candidate":
config["code_commit"] = "1234567890"
path.write_text(yaml.safe_dump(config))
elif change == "dataset":
config["benchmark"]["case_ids"] = ["1"]
path.write_text(yaml.safe_dump(config))
elif change == "runtime":
(home / "code/runtime/reme/__init__.py").write_text("tampered")
elif change == "test_code":
(home / "code/repo/plugins/beam/src/reme_beam/__init__.py").write_text("tampered")
else:
file = next((home / "datasets/training-set/cases").glob("*.json"))
case = read_json(file)
case["queries"][0]["question"] = "tampered"
file.write_text(json.dumps(case))
before = snapshot(home)
with pytest.raises((ValueError, CalledProcessError)):
await home_eval.run(path, prepare_only=True)
assert snapshot(home) == before
@pytest.mark.asyncio
@pytest.mark.parametrize("location", ["same", "child", "parent"])
async def test_rejects_overlapping_workspaces_before_start(app, inputs, tmp_path, monkeypatch, location):
_, commit = await prepare_source(app, inputs)
path, config = write_config(tmp_path, app, inputs[0], commit)
source = Path(app.config.workspace_dir)
target = {"same": source, "child": source / "test", "parent": source.parent}[location]
config["test_workspace"] = str(target)
path.write_text(yaml.safe_dump(config))
factory = AsyncMock()
monkeypatch.setattr(home_eval, "Application", factory)
with pytest.raises(ValueError, match="non-nested"):
await home_eval.run(path)
factory.assert_not_called()
@pytest.mark.asyncio
async def test_import_runtime_drift_rolls_back_and_allows_retry(app, inputs, tmp_path, monkeypatch):
source_home, commit = await prepare_source(app, inputs)
path, config = write_config(tmp_path, app, inputs[0], commit)
install = EvaluationSource.install
def fail_install(self, runtime, candidate):
runtime.mkdir()
(runtime / "partial").write_text("partial import")
raise ValueError("Source fixed runtime changed during import")
monkeypatch.setattr(EvaluationSource, "install", fail_install)
with pytest.raises(ValueError, match="during import"):
await home_eval.run(path, prepare_only=True)
target = Path(config["test_workspace"]) / "home"
assert not (target / "metadata/preparation.json").exists()
assert not list((target / "code/runtime").iterdir())
assert not list((target / "code/repo").iterdir())
monkeypatch.setattr(EvaluationSource, "install", install)
await home_eval.run(path, prepare_only=True)
assert tree_fingerprint(target / "code/runtime") == tree_fingerprint(source_home / "code/runtime")
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
@pytest.mark.asyncio
async def test_benchmark_entry_points_select_correct_defaults(benchmark, monkeypatch):
module = import_module(f"benchmark.{benchmark}.run_home_eval")
runner = AsyncMock(return_value={"status": "completed"})
monkeypatch.setattr(module, "run_evaluation", runner)
assert await module.run(commit="abcdef0") == {"status": "completed"}
runner.assert_awaited_once_with(module.DEFAULT_CONFIG, expected_benchmark=benchmark, commit="abcdef0")
@pytest.mark.parametrize("options", [{"sampling": True}, {"record_history": True}, {"max_validations": 1}])
def test_test_runner_rejects_optimizer_modes(options):
with pytest.raises(ValueError):
home_eval.TestRunConfig.model_validate(
{
"source_workspace": "/source",
"test_workspace": "/test",
"code_commit": "abcdef0",
"benchmark": {"name": "beam", "source": "/data", "variant": "100K", "all_cases": True},
"evaluation": {"record_history": False, **options},
},
)

View file

@ -7,6 +7,7 @@ import asyncio
import json
from agentscope.agent import Agent, ContextConfig
from agentscope.exception import StructuredOutputError
from agentscope.message import AssistantMsg, TextBlock, UserMsg, Usage
from agentscope.state import AgentState
import jsonschema
@ -45,12 +46,19 @@ def make_agent(tmp_path, ratio=0.9, trigger=0.8, failure=None, summary="existing
@pytest.mark.asyncio
@pytest.mark.parametrize("failure_type", [RuntimeError, StructuredOutputError])
@pytest.mark.parametrize(
("trigger", "ratio", "expected"),
[(0.8, 0.79, False), (0.8, 0.85, False), (0.8, 0.9, True), (0.65, 0.82, False), (0.65, 0.825, True)],
)
async def test_real_compression_failure_uses_dev_meta_emergency_threshold(tmp_path, trigger, ratio, expected):
agent = make_agent(tmp_path, ratio, trigger)
async def test_real_compression_failure_uses_dev_meta_emergency_threshold(
tmp_path,
trigger,
ratio,
expected,
failure_type,
):
agent = make_agent(tmp_path, ratio, trigger, failure=failure_type("Invalid structured output"))
context_before, summary_before = agent.state.context, agent.state.summary
await agent.compress_context()
assert agent.validation_cycle_timer.context_compression_count == int(expected)
@ -63,10 +71,16 @@ async def test_real_compression_failure_uses_dev_meta_emergency_threshold(tmp_pa
@pytest.mark.asyncio
@pytest.mark.parametrize("failure_type", [RuntimeError, StructuredOutputError])
@pytest.mark.parametrize("summary", ["existing summary", [TextBlock(text="existing summary")]])
@pytest.mark.parametrize("reserve_ratio", [0.2, 0])
async def test_offload_retains_full_messages_summary_usage_and_recent_context(tmp_path, summary, reserve_ratio):
agent = make_agent(tmp_path, summary=summary)
async def test_offload_retains_full_messages_summary_usage_and_recent_context(
tmp_path,
summary,
reserve_ratio,
failure_type,
):
agent = make_agent(tmp_path, summary=summary, failure=failure_type("Invalid structured output"))
agent.context_config.reserve_ratio = reserve_ratio
agent.state.context[-1] = AssistantMsg(
id=agent.state.reply_id,
@ -87,7 +101,7 @@ async def test_offload_retains_full_messages_summary_usage_and_recent_context(tm
assert data["session_id"] == agent.state.session_id
assert data["context_ratio"] == 0.9
assert data["emergency_ratio"] == 0.9
assert data["failure"]["type"] == "RuntimeError"
assert data["failure"]["type"] == failure_type.__name__
text = agent.state.summary
if isinstance(text, list):
text = "\n".join(block.text for block in text)
@ -117,6 +131,7 @@ async def test_unrecoverable_failures_and_cancellation_propagate(tmp_path, failu
("failure", "expected"),
[
(RuntimeError("provider failed"), True),
(StructuredOutputError("invalid summary"), True),
(RuntimeError("The system prompt exceeds threshold"), False),
(jsonschema.ValidationError("invalid summary"), True),
(openai.OpenAIError("provider unavailable"), True),