diff --git a/benchmark/beam/README.md b/benchmark/beam/README.md index 9045b865..0fc26f72 100644 --- a/benchmark/beam/README.md +++ b/benchmark/beam/README.md @@ -192,3 +192,39 @@ from benchmark.beam.run_home import run result = await run(workspace_dir="/path/to/new-workspace") ``` + + +## Evaluate a candidate on an independent test set + +The default evaluates every 100K case, matching the previous standalone evaluation. This is an independent +workspace, not a disjoint held-out split. Select another `variant`, or replace `all_cases: true` with +`case_ids: [...]`, in [config_home_eval.yaml](config_home_eval.yaml) for a different test set. + +The entry point imports a full candidate SHA or unambiguous hexadecimal prefix from an existing Home +optimization workspace and copies its prepared fixed runtime. It does not require the original ReMe source +repository or change the optimizer's checkout. Paths in the configuration are repository-relative. + +Prepare first, then use the service from two terminals (replace `CANDIDATE_SHA`): + +```bash +python benchmark/beam/run_home_eval.py --commit CANDIDATE_SHA --prepare-only +# Terminal 1: +reme start workspace_dir=benchmark/beam/workspaces/home-test-100K 'plugins=["home","beam-judge"]' +# Terminal 2: +reme home_evaluate workspace_dir=benchmark/beam/workspaces/home-test-100K record_history=false +``` + +### Local one-shot test evaluation + +```bash +python benchmark/beam/run_home_eval.py --commit CANDIDATE_SHA +# Optional overrides: --config /path/to/test.yaml --source-workspace /path/to/search --test-workspace /path/to/test +``` + +Python callers can use `await benchmark.beam.run_home_eval.run(commit="CANDIDATE_SHA")`. +The runner rebuilds construction and scores every selected query with sampling and history recording disabled. +It uses the existing Docker evaluation runtime and host judge configuration; model credentials are required +for evaluation. Results stay in the test workspace's `home/evaluations/`, with original candidate provenance +in each manifest's `test_evaluation`. The exported snapshot has its own local `evaluation_commit_sha`. +Rerunning the same request reopens unchanged prepared inputs and creates another evaluation result. Use a +new test workspace for another candidate or selection. Source and test workspaces must not overlap. diff --git a/benchmark/beam/README_ZH.md b/benchmark/beam/README_ZH.md index 42bd4ea6..a10ed627 100644 --- a/benchmark/beam/README_ZH.md +++ b/benchmark/beam/README_ZH.md @@ -172,3 +172,38 @@ python benchmark/beam/run_home.py --optimize 不加 `--optimize` 时仍只准备。 默认选择 BEAM **100K 全部 case**。 + + +## 独立测试集评测 + +默认在独立工作区评测 **100K 全部 case**,与旧独立评测入口一致,不代表与训练集互斥的留出集。 +可在 [config_home_eval.yaml](config_home_eval.yaml) 更换 `variant`,或把 `all_cases: true` +替换为 `case_ids: [...]`,指定测试数据。 + +入口从已有 Home 优化工作区导出指定候选提交,接受完整 SHA 或无歧义的十六进制短 SHA, +并复制该实验已准备的固定 runtime。无需原始 ReMe 源码仓库,也不会切换或修改优化工作区。 +配置内路径均相对仓库根目录。 + +先准备测试工作区,再分别在两个终端启动服务和调用评测(替换 `CANDIDATE_SHA`): + +```bash +python benchmark/beam/run_home_eval.py --commit CANDIDATE_SHA --prepare-only +# 终端 1: +reme start workspace_dir=benchmark/beam/workspaces/home-test-100K 'plugins=["home","beam-judge"]' +# 终端 2: +reme home_evaluate workspace_dir=benchmark/beam/workspaces/home-test-100K record_history=false +``` + +### 本地一次性测试评测 + +```bash +python benchmark/beam/run_home_eval.py --commit CANDIDATE_SHA +# 可选:--config /path/to/test.yaml --source-workspace /path/to/search --test-workspace /path/to/test +``` + +Python 可调用 `await benchmark.beam.run_home_eval.run(commit="CANDIDATE_SHA")`。 +入口完整重建记忆并对所选 case 的全部 query 评分,关闭采样与优化历史记录,使用现有 Docker +评测环境和宿主 judge 配置;实际评测需要模型凭证。结果保存在测试工作区的 `home/evaluations/`, +manifest 的 `test_evaluation` 记录来源候选,导出后的本地提交另记为 `evaluation_commit_sha`。 +相同请求可复用未改动的准备结果并新增一次评测;更换候选或数据选择时需使用新测试工作区。 +测试工作区与来源工作区必须互不包含。 diff --git a/benchmark/beam/config_home_eval.yaml b/benchmark/beam/config_home_eval.yaml new file mode 100644 index 00000000..5a43a8bf --- /dev/null +++ b/benchmark/beam/config_home_eval.yaml @@ -0,0 +1,13 @@ +# Independent full-dataset evaluation; paths are relative to the ReMe repository. +source_workspace: benchmark/beam/workspaces/home-100K +test_workspace: benchmark/beam/workspaces/home-test-100K +code_commit: "" # Required: set a candidate SHA/prefix, or pass --commit. +benchmark: + name: beam + source: benchmark/beam/dataset/BEAM + variant: 100K + all_cases: true # Full 100K evaluation; not a disjoint held-out split. +evaluation: + concurrency: 4 + timeout_seconds: 7200 + record_history: false diff --git a/benchmark/beam/run_home_eval.py b/benchmark/beam/run_home_eval.py new file mode 100644 index 00000000..bb028e94 --- /dev/null +++ b/benchmark/beam/run_home_eval.py @@ -0,0 +1,21 @@ +"""Evaluate a BEAM Home candidate in an independent test workspace.""" + +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from benchmark.home_eval import main, run as run_evaluation # noqa: E402 # pylint: disable=wrong-import-position + +DEFAULT_CONFIG = Path(__file__).with_name("config_home_eval.yaml") + + +async def run(config_path=DEFAULT_CONFIG, **kwargs): + """Score the selected BEAM commit without changing its optimization workspace.""" + return await run_evaluation(config_path, expected_benchmark="beam", **kwargs) + + +if __name__ == "__main__": + main("beam", DEFAULT_CONFIG) diff --git a/benchmark/home_eval.py b/benchmark/home_eval.py new file mode 100644 index 00000000..8c4174ed --- /dev/null +++ b/benchmark/home_eval.py @@ -0,0 +1,207 @@ +"""Standalone test evaluation shared by the BEAM and LongMemEval entry points.""" + +import argparse +import asyncio +import json +from pathlib import Path + +from pydantic import Field, model_validator +import yaml + +from reme_home.code.repository import resolve_revision, validate_repository +from reme_home.evaluation.artifacts import contained, read_json +from reme_home.evaluation.component import blocking +from reme_home.evaluation.inputs import EvaluationSpec, load_inputs +from reme_home.judge import JUDGE_PLUGINS +from reme_home.preparation.code import tree_fingerprint +from reme_home.preparation.data.models import fingerprint +from reme_home.preparation.evaluation_source import EvaluationSource +from reme_home.preparation.spec import BenchmarkSpec, DebugSpec, SpecModel + +from reme.application import Application + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + + +class TestRunConfig(SpecModel): + """Explicit dataset selection and execution options for one pinned candidate.""" + + source_workspace: str = Field(min_length=1) + test_workspace: str = Field(min_length=1) + code_commit: str = Field(min_length=4) + benchmark: BenchmarkSpec + exclude_training_cases: bool = False + evaluation: EvaluationSpec = Field(default_factory=lambda: EvaluationSpec(record_history=False)) + + @model_validator(mode="after") + def validate_test_mode(self): + """Test runs score every prepared query and never consume optimizer budgets.""" + options = self.evaluation + if any( + ( + options.debug, + options.sampling, + options.record_history, + options.max_validations is not None, + options.sampling_policy is not None, + options.case_ids is not None, + options.query_ids is not None, + options.reuse_construction_from is not None, + ), + ): + raise ValueError( + "Test evaluation requires full scoring with record_history=false; select cases in benchmark", + ) + if self.exclude_training_cases and (self.benchmark.name != "longmemeval" or not self.benchmark.all_cases): + raise ValueError("exclude_training_cases requires LongMemEval with all_cases=true") + return self + + +def project_path(value: str, project_root: Path) -> Path: + """Resolve runner paths independently of the launching working directory.""" + if value.startswith("~"): + raise ValueError("Use absolute or project-relative paths, not home-relative paths") + return (project_root / value).resolve() + + +def load_request(config_path, *, commit=None, source_workspace=None, test_workspace=None, project_root=PROJECT_ROOT): + """Validate all source inputs before starting an application or writing test data.""" + raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8")) + for key, value in ( + ("code_commit", commit), + ("source_workspace", source_workspace), + ("test_workspace", test_workspace), + ): + if value is not None: + raw[key] = value + # Execution settings can be partial while retaining the test-only history default. + raw["evaluation"] = {"record_history": False, **raw.get("evaluation", {})} + config = TestRunConfig.model_validate(raw) + source_path = project_path(config.source_workspace, project_root) + target = project_path(config.test_workspace, project_root) + if target.is_relative_to(source_path) or source_path.is_relative_to(target): + raise ValueError("Test and source workspaces must be separate, non-nested directories") + if target.exists() and ( + not target.is_dir() or (any(target.iterdir()) and not (target / "home/metadata/workspace.json").is_file()) + ): + raise ValueError("Use an empty test workspace or a compatible prepared Home test workspace") + source = EvaluationSource.load(source_path, config.code_commit) + benchmark = config.benchmark.model_copy(deep=True) + if benchmark.name != source.spec.benchmark.name: + raise ValueError("Test benchmark must match the source experiment") + benchmark.source = str(project_path(benchmark.source, project_root).resolve(strict=True)) + if benchmark.evidence_source is not None: + benchmark.evidence_source = str(project_path(benchmark.evidence_source, project_root).resolve(strict=True)) + if config.exclude_training_cases: + items = json.loads(Path(benchmark.source).read_text(encoding="utf-8")) + ids = [str(item["question_id"]) for item in items] + if len(ids) != len(set(ids)): + raise ValueError("Duplicate LongMemEval question IDs") + training = set(source.record["dataset"]["case_ids"]) + if training - set(ids): + raise ValueError("Test dataset is missing training IDs; provide the original full dataset") + benchmark.case_ids = [case_id for case_id in ids if case_id not in training] + benchmark.all_cases = False + benchmark = BenchmarkSpec.model_validate(benchmark.model_dump()) + spec = source.spec.model_copy(deep=True) + spec.benchmark = benchmark + spec.debug = DebugSpec(enabled=False) + return config, target, source, spec + + +def existing_preparation(home, source, spec): + """Reopen only the same request with unchanged code, runtime and normalized data.""" + marker = contained(home, "metadata/preparation.json") + if not marker.exists(): + return None + record = read_json(marker) + expected = {**source.provenance, "request_fingerprint": fingerprint(spec.model_dump(mode="json"))} + if record.get("test_evaluation") != expected: + raise ValueError("Test workspace belongs to a different candidate or dataset; use a new workspace") + domain = yaml.safe_load(contained(home, "domain_spec.yaml").read_text(encoding="utf-8")) + if fingerprint(domain) != record["config_fingerprint"]: + raise ValueError("Test preparation configuration has changed") + repository, _ = validate_repository(home, record) + if ( + resolve_revision(repository, "HEAD") != record["code"]["initial_commit"] + or tree_fingerprint(repository) != record["code"]["candidate_fingerprint"] + or tree_fingerprint(contained(home, "code/runtime", directory=True)) != source.provenance["runtime_fingerprint"] + ): + raise ValueError("Prepared test code or runtime has changed") + load_inputs(home, EvaluationSpec(record_history=False)) + return record + + +async def run( + config_path, + *, + commit=None, + source_workspace=None, + test_workspace=None, + prepare_only=False, + expected_benchmark=None, +): + """Prepare transactionally, then score through the normal application lifecycle.""" + config, target, source, spec = await blocking( + load_request, + config_path, + commit=commit, + source_workspace=source_workspace, + test_workspace=test_workspace, + ) + if expected_benchmark is not None and config.benchmark.name != expected_benchmark: + raise ValueError(f"This entry point requires benchmark.name={expected_benchmark}") + app = Application( + workspace_dir=str(target), + plugins=["home", JUDGE_PLUGINS[config.benchmark.name]], + service={"backend": "cli"}, + enable_logo=False, + log_to_file=False, + ) + try: + await app.start() + home = target / "home" + prepared = await blocking(existing_preparation, home, source, spec) + if prepared is None: + preparation = app.context.components["home.preparation"]["default"] + prepared = await preparation.prepare(spec=spec.model_dump(mode="json"), code_source=source) + result = { + "test_workspace": str(target), + **source.provenance, + "evaluation_commit_sha": prepared["code"]["initial_commit"], + "case_count": prepared["dataset"]["case_count"], + "query_count": prepared["dataset"]["query_count"], + } + if not prepare_only: + response = await app.run_job( + "home_evaluate", + **config.evaluation.model_dump(mode="json", exclude_unset=True), + ) + if not response.success: + raise RuntimeError(response.answer) + result["evaluation"] = response.answer + return result + finally: + await app.close() + + +def main(benchmark, default_config): + """Explicit local one-shot entry point, with preparation-only service setup.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=default_config) + parser.add_argument("--commit", help="Candidate commit SHA or unambiguous hexadecimal prefix") + parser.add_argument("--source-workspace", help="Prepared optimization workspace") + parser.add_argument("--test-workspace", help="Independent test output workspace") + parser.add_argument("--prepare-only", action="store_true", help="Prepare for a later home_evaluate service call") + args = parser.parse_args() + result = asyncio.run( + run( + args.config, + commit=args.commit, + source_workspace=args.source_workspace, + test_workspace=args.test_workspace, + prepare_only=args.prepare_only, + expected_benchmark=benchmark, + ), + ) + print(json.dumps(result, ensure_ascii=False, indent=2)) diff --git a/benchmark/longmemeval/README.md b/benchmark/longmemeval/README.md index 71467a90..e663301c 100644 --- a/benchmark/longmemeval/README.md +++ b/benchmark/longmemeval/README.md @@ -174,3 +174,40 @@ out. The runner reads the IDs directly from YAML without loading group files. [home_evidence_sessions.jsonl](home_evidence_sessions.jsonl) is a byte-for-byte copy of that revision's `dataset/longmemeval_evidence_sessions.jsonl`; reviewed evidence-session annotations are preserved during normalization. + + +## Evaluate a candidate on an independent test set + +The default excludes the source experiment's actual training IDs from the full dataset. With the default +250-case training workspace this selects held-out groups 6–10. For an explicit selection, set +`exclude_training_cases: false` and replace `all_cases: true` with `case_ids: [...]` in +[config_home_eval.yaml](config_home_eval.yaml). An empty held-out selection fails before preparation. + +The entry point imports a full candidate SHA or unambiguous hexadecimal prefix from an existing Home +optimization workspace and copies its prepared fixed runtime. It does not require the original ReMe source +repository or change the optimizer's checkout. Paths in the configuration are repository-relative. + +Prepare first, then use the service from two terminals (replace `CANDIDATE_SHA`): + +```bash +python benchmark/longmemeval/run_home_eval.py --commit CANDIDATE_SHA --prepare-only +# Terminal 1: +reme start workspace_dir=benchmark/longmemeval/workspaces/home-test-held-out 'plugins=["home","lme-judge"]' +# Terminal 2: +reme home_evaluate workspace_dir=benchmark/longmemeval/workspaces/home-test-held-out record_history=false +``` + +### Local one-shot test evaluation + +```bash +python benchmark/longmemeval/run_home_eval.py --commit CANDIDATE_SHA +# Optional overrides: --config /path/to/test.yaml --source-workspace /path/to/search --test-workspace /path/to/test +``` + +Python callers can use `await benchmark.longmemeval.run_home_eval.run(commit="CANDIDATE_SHA")`. +The runner rebuilds construction and scores every selected query with sampling and history recording disabled. +It uses the existing Docker evaluation runtime and host judge configuration; model credentials are required +for evaluation. Results stay in the test workspace's `home/evaluations/`, with original candidate provenance +in each manifest's `test_evaluation`. The exported snapshot has its own local `evaluation_commit_sha`. +Rerunning the same request reopens unchanged prepared inputs and creates another evaluation result. Use a +new test workspace for another candidate or selection. Source and test workspaces must not overlap. diff --git a/benchmark/longmemeval/README_ZH.md b/benchmark/longmemeval/README_ZH.md index 7b93b7ee..b843179c 100644 --- a/benchmark/longmemeval/README_ZH.md +++ b/benchmark/longmemeval/README_ZH.md @@ -147,3 +147,39 @@ ID 已按原顺序直接写入 [config_home.yaml](config_home.yaml) 的 `benchma 运行时无需读取分组文件,group_6–group_10 保持为留出集。 独立审阅的证据标注保存在 [home_evidence_sessions.jsonl](home_evidence_sessions.jsonl),来源见英文说明。 + + +## 独立测试集评测 + +默认从完整数据中排除来源工作区实际使用的训练 ID;使用默认 250 个训练 case 时,测试集为 +**group_6–group_10,共 250 个 case**。若要显式选取测试 case,可在 +[config_home_eval.yaml](config_home_eval.yaml) 设置 `exclude_training_cases: false`, +并把 `all_cases: true` 替换为 `case_ids: [...]`。留出集为空时会直接报错。 + +入口从已有 Home 优化工作区导出指定候选提交,接受完整 SHA 或无歧义的十六进制短 SHA, +并复制该实验已准备的固定 runtime。无需原始 ReMe 源码仓库,也不会切换或修改优化工作区。 +配置内路径均相对仓库根目录。 + +先准备测试工作区,再分别在两个终端启动服务和调用评测(替换 `CANDIDATE_SHA`): + +```bash +python benchmark/longmemeval/run_home_eval.py --commit CANDIDATE_SHA --prepare-only +# 终端 1: +reme start workspace_dir=benchmark/longmemeval/workspaces/home-test-held-out 'plugins=["home","lme-judge"]' +# 终端 2: +reme home_evaluate workspace_dir=benchmark/longmemeval/workspaces/home-test-held-out record_history=false +``` + +### 本地一次性测试评测 + +```bash +python benchmark/longmemeval/run_home_eval.py --commit CANDIDATE_SHA +# 可选:--config /path/to/test.yaml --source-workspace /path/to/search --test-workspace /path/to/test +``` + +Python 可调用 `await benchmark.longmemeval.run_home_eval.run(commit="CANDIDATE_SHA")`。 +入口完整重建记忆并对所选 case 的全部 query 评分,关闭采样与优化历史记录,使用现有 Docker +评测环境和宿主 judge 配置;实际评测需要模型凭证。结果保存在测试工作区的 `home/evaluations/`, +manifest 的 `test_evaluation` 记录来源候选,导出后的本地提交另记为 `evaluation_commit_sha`。 +相同请求可复用未改动的准备结果并新增一次评测;更换候选或数据选择时需使用新测试工作区。 +测试工作区与来源工作区必须互不包含。 diff --git a/benchmark/longmemeval/config_home_eval.yaml b/benchmark/longmemeval/config_home_eval.yaml new file mode 100644 index 00000000..66cf7598 --- /dev/null +++ b/benchmark/longmemeval/config_home_eval.yaml @@ -0,0 +1,14 @@ +# Evaluate the complement of the source experiment's training IDs in the full dataset. +# With the default training workspace this selects held-out groups 6-10 (250 cases). +source_workspace: benchmark/longmemeval/workspaces/home-groups-1-5 +test_workspace: benchmark/longmemeval/workspaces/home-test-held-out +code_commit: "" # Required: set a candidate SHA/prefix, or pass --commit. +exclude_training_cases: true +benchmark: + name: longmemeval + source: benchmark/longmemeval/dataset/longmemeval_s_reme_cleaned.json + all_cases: true +evaluation: + concurrency: 4 + timeout_seconds: 7200 + record_history: false diff --git a/benchmark/longmemeval/run_home_eval.py b/benchmark/longmemeval/run_home_eval.py new file mode 100644 index 00000000..bb5ec3ee --- /dev/null +++ b/benchmark/longmemeval/run_home_eval.py @@ -0,0 +1,21 @@ +"""Evaluate a LongMemEval Home candidate on an independent test dataset.""" + +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from benchmark.home_eval import main, run as run_evaluation # noqa: E402 # pylint: disable=wrong-import-position + +DEFAULT_CONFIG = Path(__file__).with_name("config_home_eval.yaml") + + +async def run(config_path=DEFAULT_CONFIG, **kwargs): + """Score the selected LongMemEval commit in its own Home workspace.""" + return await run_evaluation(config_path, expected_benchmark="longmemeval", **kwargs) + + +if __name__ == "__main__": + main("longmemeval", DEFAULT_CONFIG) diff --git a/plugins/home/src/reme_home/agent/compression.py b/plugins/home/src/reme_home/agent/compression.py index 0241611e..caaf3100 100644 --- a/plugins/home/src/reme_home/agent/compression.py +++ b/plugins/home/src/reme_home/agent/compression.py @@ -5,7 +5,7 @@ import logging from pathlib import Path from uuid import uuid4 -from agentscope.exception import ToolJSONDecodeError +from agentscope.exception import StructuredOutputError, ToolJSONDecodeError from agentscope.message import TextBlock import jsonschema import openai @@ -67,7 +67,16 @@ class RecoverableCompressionAgent(BoundedReActAgent): """Match dev_meta's model/output failures, excluding uncompressible prompts.""" if isinstance(exc, RuntimeError): return not str(exc).startswith("The system prompt") - return isinstance(exc, (ToolJSONDecodeError, jsonschema.ValidationError, ValidationError, openai.OpenAIError)) + return isinstance( + exc, + ( + StructuredOutputError, + ToolJSONDecodeError, + jsonschema.ValidationError, + ValidationError, + openai.OpenAIError, + ), + ) async def _fallback_compress_context(self, cause, context_ratio, config): """Persist older messages before replacing state; never call a second model.""" diff --git a/plugins/home/src/reme_home/evaluation/component.py b/plugins/home/src/reme_home/evaluation/component.py index 342a8a1a..10cccd4c 100644 --- a/plugins/home/src/reme_home/evaluation/component.py +++ b/plugins/home/src/reme_home/evaluation/component.py @@ -291,6 +291,8 @@ class HomeEvaluation(BaseComponent): } if source is not None: manifest["construction_source"] = source + if "test_evaluation" in record: + manifest["test_evaluation"] = record["test_evaluation"] if admission is not None: manifest["candidate_gate"] = admission if reuse is not None: diff --git a/plugins/home/src/reme_home/preparation/component.py b/plugins/home/src/reme_home/preparation/component.py index d0cda079..67a2d66a 100644 --- a/plugins/home/src/reme_home/preparation/component.py +++ b/plugins/home/src/reme_home/preparation/component.py @@ -14,6 +14,7 @@ from reme.components.base_component import BaseComponent from reme.steps.base_step import BaseStep, Ref from .code import prepare_code +from .evaluation_source import EvaluationSource from .data import prepare_dataset from .data.basic import derive_debug_cases from .data.models import fingerprint @@ -58,10 +59,18 @@ class HomePreparation(BaseComponent): if self._pending: await asyncio.gather(*self._pending, return_exceptions=True) - async def prepare(self, *, spec: dict | None = None, spec_path: str | None = None) -> dict: + async def prepare( + self, + *, + spec: dict | None = None, + spec_path: str | None = None, + code_source: EvaluationSource | None = None, + ) -> dict: """Keep blocking data/Git work off the event loop and await rollback on cancellation.""" stop = Event() - task = asyncio.create_task(asyncio.to_thread(self._prepare, spec=spec, spec_path=spec_path, stop=stop)) + task = asyncio.create_task( + asyncio.to_thread(self._prepare, spec=spec, spec_path=spec_path, stop=stop, code_source=code_source), + ) self._pending.add(task) cancelled = False try: @@ -80,7 +89,7 @@ class HomePreparation(BaseComponent): finally: self._pending.discard(task) - def _prepare(self, *, spec: dict | None, spec_path: str | None, stop: Event) -> dict: + def _prepare(self, *, spec: dict | None, spec_path: str | None, stop: Event, code_source=None) -> dict: """Prepare a started application's workspace from exactly one input form. Local YAML paths must stay inside the workspace. Explicit dataset/repository @@ -108,19 +117,23 @@ class HomePreparation(BaseComponent): self.judge.require_benchmark(config.benchmark.name) # Resolve explicitly supplied source inputs before creating transaction state. source = _source_path(workspace, config.benchmark.source) - repository = _source_path(workspace, config.runtime.source_repo) + repository = _source_path(workspace, config.runtime.source_repo) if code_source is None else None evidence = ( _source_path(workspace, config.benchmark.evidence_source) if config.benchmark.evidence_source else None ) config.benchmark.source = str(source) - config.runtime.source_repo = str(repository) + if repository is not None: + config.runtime.source_repo = str(repository) config.benchmark.evidence_source = str(evidence) if evidence else None existed = check_targets(root, state) transaction = state / TRANSACTION transaction.mkdir() try: sync_directory(state) - result = self._stage(config, transaction, source, repository, evidence) + if code_source is None: + result = self._stage(config, transaction, source, repository, evidence) + else: + result = self._stage(config, transaction, source, repository, evidence, code_source=code_source) def check_cancelled(): if stop.is_set(): @@ -142,7 +155,8 @@ class HomePreparation(BaseComponent): return result @staticmethod - def _stage(config, transaction, source, repository, evidence) -> dict: + def _stage(config, transaction, source, repository, evidence, *, code_source=None) -> dict: + request_fingerprint = fingerprint(config.model_dump(mode="json")) dataset = prepare_dataset( transaction / "output-2", config.benchmark.name, @@ -163,7 +177,11 @@ class HomePreparation(BaseComponent): ) else: (transaction / "output-3/cases").mkdir(parents=True) - code = prepare_code(config, repository, transaction / "output-0", transaction / "output-1") + code = ( + prepare_code(config, repository, transaction / "output-0", transaction / "output-1") + if code_source is None + else code_source.install(transaction / "output-0", transaction / "output-1") + ) config.runtime.source_ref = code["source_commit"] domain = config.model_dump(mode="json") (transaction / "output-4").write_text(yaml.safe_dump(domain, sort_keys=False), encoding="utf-8") @@ -179,6 +197,11 @@ class HomePreparation(BaseComponent): "dataset": dataset.model_dump(mode="json"), "code": code, } + if code_source is not None: + result["test_evaluation"] = { + **code_source.provenance, + "request_fingerprint": request_fingerprint, + } write_json(transaction / "output-6", result) return result diff --git a/plugins/home/src/reme_home/preparation/evaluation_source.py b/plugins/home/src/reme_home/preparation/evaluation_source.py new file mode 100644 index 00000000..f2652df1 --- /dev/null +++ b/plugins/home/src/reme_home/preparation/evaluation_source.py @@ -0,0 +1,87 @@ +"""Read-only import of a pinned Home candidate and its fixed runtime.""" + +from dataclasses import dataclass +from pathlib import Path +import re +import shutil + +import yaml + +from ..code.repository import resolve_revision, validate_repository, validate_tree +from ..code.snapshot import export_blob +from ..evaluation.artifacts import contained, read_json +from .code import git, tree_fingerprint +from .data.models import fingerprint +from .spec import PreparationSpec, relative_path + + +@dataclass(frozen=True) +class EvaluationSource: + """Host-owned source identity; imports never change the source checkout.""" + + home: Path + commit: str + spec: PreparationSpec + record: dict + + @classmethod + def load(cls, workspace: Path, commit: str): + """Require a prepared workspace and an unambiguous commit SHA or prefix.""" + if re.fullmatch(r"[0-9a-fA-F]{4,64}", commit) is None: + raise ValueError("code_commit must be a full commit SHA or hexadecimal prefix") + home = contained(workspace, "home", directory=True) + record = read_json(contained(home, "metadata/preparation.json")) + domain = yaml.safe_load(contained(home, "domain_spec.yaml").read_text(encoding="utf-8")) + if fingerprint(domain) != record["config_fingerprint"]: + raise ValueError("Source preparation configuration has changed") + spec = PreparationSpec.model_validate(domain) + repository, _ = validate_repository(home, record) + revision = resolve_revision(repository, commit) + git(repository, "merge-base", "--is-ancestor", record["code"]["initial_commit"], revision) + validate_tree(repository, record, revision) + runtime = contained(home, "code/runtime", directory=True) + if tree_fingerprint(runtime) != record["code"]["runtime_fingerprint"]: + raise ValueError("Source fixed runtime has changed since preparation") + return cls(home, revision, spec, record) + + @property + def provenance(self): + """Stable identity checked when reopening a standalone test workspace.""" + return { + "source_workspace": str(self.home.parent), + "source_preparation_id": self.record["preparation_id"], + "source_commit_sha": self.commit, + "runtime_fingerprint": self.record["code"]["runtime_fingerprint"], + } + + def install(self, runtime: Path, candidate: Path) -> dict: + """Copy fixed sources and export regular Git blobs, preserving executable bits.""" + source = contained(self.home, "code/runtime", directory=True) + expected = self.record["code"]["runtime_fingerprint"] + if tree_fingerprint(source) != expected: + raise ValueError("Source fixed runtime changed before import") + shutil.copytree(source, runtime) + if tree_fingerprint(runtime) != expected or tree_fingerprint(source) != expected: + raise ValueError("Source fixed runtime changed during import") + repository = contained(self.home, "code/repo", directory=True) + candidate.mkdir() + entries = git(repository, "ls-tree", "-r", "-z", "--full-tree", self.commit, strip=False) + for entry in filter(None, entries.split("\0")): + header, name = entry.split("\t", 1) + mode, kind, oid = header.split() + path = Path(relative_path(name)) + if mode not in {"100644", "100755"} or kind != "blob" or ".git" in path.parts: + raise ValueError(f"Candidate must contain regular files only: {name}") + destination = candidate / path + destination.parent.mkdir(parents=True, exist_ok=True) + export_blob(repository, oid, destination) + destination.chmod(0o755 if mode == "100755" else 0o644) + # The isolated repository's init is the selected snapshot, not the optimizer baseline. + git(candidate, "init", "--initial-branch=init", "--template=") + git(candidate, "add", "--force", "--all") + git(candidate, "-c", "user.name=Home", "-c", "user.email=home@localhost", "commit", "-m", "Test candidate") + return { + **self.record["code"], + "initial_commit": resolve_revision(candidate, "HEAD"), + "candidate_fingerprint": tree_fingerprint(candidate), + } diff --git a/plugins/home/tests/test_home_eval.py b/plugins/home/tests/test_home_eval.py new file mode 100644 index 00000000..f5d53c2b --- /dev/null +++ b/plugins/home/tests/test_home_eval.py @@ -0,0 +1,283 @@ +"""Standalone test workspaces use pinned sources and isolated evaluation lifecycles.""" + +# pylint: disable=missing-function-docstring,unused-argument,protected-access,redefined-outer-name + +import copy +from importlib import import_module +import json +from pathlib import Path +from subprocess import CalledProcessError +from unittest.mock import AsyncMock + +import pytest +import yaml + +from reme_home.evaluation.artifacts import read_json +from reme_home.evaluation.inputs import EvaluationSpec, load_inputs +from reme_home.preparation.code import tree_fingerprint +from reme_home.preparation.evaluation_source import EvaluationSource +from reme_home.evaluation import component as evaluation + +from test_evaluation import FakeFactory + +from test_preparation import ( # noqa: F401 # pylint: disable=unused-import + command, + home_application, + preparation_inputs, + snapshot, +) + +from benchmark import home_eval +from reme.schema import Response +from reme.components.agent_wrapper import AsAgentWrapper + + +def write_config(tmp_path, app, spec, commit): + config = { + "source_workspace": str(app.config.workspace_dir), + "test_workspace": str(tmp_path / "test-workspace"), + "code_commit": commit, + "benchmark": copy.deepcopy(spec["benchmark"]), + "evaluation": {"concurrency": 2, "timeout_seconds": 60}, + } + path = tmp_path / "test.yaml" + path.write_text(yaml.safe_dump(config)) + return path, config + + +async def prepare_source(app, inputs): + spec, _ = inputs + response = await app.run_job("home_prepare", spec=spec) + assert response.success, response.answer + home = Path(app.config.workspace_dir) / "home" + return home, response.answer["code"]["initial_commit"] + + +@pytest.mark.asyncio +async def test_import_historical_commit_ignores_dirty_checkout_and_missing_original_repo(app, inputs, tmp_path): + home, _ = await prepare_source(app, inputs) + repo = home / "code/repo" + command(repo, "switch", "-c", "candidate") + file = repo / "plugins/beam/src/reme_beam/__init__.py" + file.write_text("value = 2\n") + file.chmod(0o755) + command(repo, "add", ".") + command(repo, "-c", "user.name=Test", "-c", "user.email=test@localhost", "commit", "-m", "candidate") + selected = command(repo, "rev-parse", "HEAD") + file.write_text("value = 3\n") + (repo / "untracked.txt").write_text("must not export") + spec, _ = inputs + original = Path(spec["runtime"]["source_repo"]) + original.rename(tmp_path / "original-unavailable") + path, _ = write_config(tmp_path, app, spec, selected[:10]) + before = snapshot(home) + + result = await home_eval.run(path, prepare_only=True) + + assert result["source_commit_sha"] == selected + target = Path(result["test_workspace"]) / "home" + imported = target / "code/repo/plugins/beam/src/reme_beam/__init__.py" + assert imported.read_text() == "value = 2\n" + assert imported.stat().st_mode & 0o111 + assert not (target / "code/repo/untracked.txt").exists() + assert command(target / "code/repo", "branch", "--show-current") == "init" + assert tree_fingerprint(target / "code/runtime") == tree_fingerprint(home / "code/runtime") + assert read_json(target / "metadata/preparation.json")["test_evaluation"]["source_commit_sha"] == selected + assert not (target / "datasets/debug-cases/manifest.json").exists() + assert snapshot(home) == before + + +@pytest.mark.asyncio +async def test_reopen_scores_all_queries_without_optimizer_history_and_closes(app, inputs, tmp_path, monkeypatch): + home, commit = await prepare_source(app, inputs) + path, _ = write_config(tmp_path, app, inputs[0], commit) + await home_eval.run(path, prepare_only=True) + applications = [] + factory = home_eval.Application + + def tracked_application(**kwargs): + application = factory(**kwargs) + application.close = AsyncMock(wraps=application.close) + + async def evaluate(name, **options): + assert name == "home_evaluate" + assert options["record_history"] is False + assert options["concurrency"] == 2 + assert not options.get("sampling") and not options.get("debug") + record, _, cases = load_inputs(Path(kwargs["workspace_dir"]) / "home", EvaluationSpec(**options)) + assert len(cases) == record["dataset"]["case_count"] == 2 + assert sum(len(case.queries) for case in cases) == 2 + return Response(answer={"result_dir": "test-results"}) + + application.run_job = AsyncMock(side_effect=evaluate) + applications.append(application) + return application + + monkeypatch.setattr(home_eval, "Application", tracked_application) + result = await home_eval.run(path) + assert result["evaluation"] == {"result_dir": "test-results"} + applications[-1].close.assert_awaited_once() + assert not list((home / "evaluations").rglob("manifest.json")) + assert not list((home / "sampling").rglob("*.json")) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("app", ["beam-judge", "lme-judge"], indirect=True) +async def test_full_evaluation_records_source_provenance_with_fake_docker(app, inputs, tmp_path, monkeypatch): + spec, lme = inputs + if "lme-judge" in app.config.plugins: + spec["benchmark"] = {"name": "longmemeval", "source": str(lme), "all_cases": True} + else: + for file in Path(spec["benchmark"]["source"]).glob("chats/*/*/probing_questions/probing_questions.json"): + data = read_json(file) + data["fact"][0]["rubric"] = ["says hello"] + file.write_text(json.dumps(data)) + home, commit = await prepare_source(app, inputs) + path, _ = write_config(tmp_path, app, spec, commit) + monkeypatch.setattr(evaluation, "DockerReMeSandboxFactory", FakeFactory) + answer = "yes" if spec["benchmark"]["name"] == "longmemeval" else '{"score": 0.5}' + monkeypatch.setattr(AsAgentWrapper, "reply", AsyncMock(return_value={"result": answer})) + before = snapshot(home) + result = await home_eval.run(path) + run = Path(result["evaluation"]["result_dir"]) + manifest = read_json(run / "manifest.json") + assert manifest["test_evaluation"]["source_commit_sha"] == commit + assert manifest["commit_sha"] == result["evaluation_commit_sha"] + assert manifest["execution_mode"] == "full_validation" + assert manifest["options"]["record_history"] is False + assert result["evaluation"]["scored_query_count"] == result["query_count"] + assert result["evaluation"]["mean_query_score"] == (1.0 if answer == "yes" else 0.5) + assert snapshot(home) == before + + +@pytest.mark.asyncio +async def test_evaluation_failure_closes_application_and_preserves_preparation(app, inputs, tmp_path, monkeypatch): + _, commit = await prepare_source(app, inputs) + path, config = write_config(tmp_path, app, inputs[0], commit) + factory = home_eval.Application + applications = [] + + def failing_application(**kwargs): + application = factory(**kwargs) + application.run_job = AsyncMock(return_value=Response(success=False, answer="Infrastructure failure")) + application.close = AsyncMock(wraps=application.close) + applications.append(application) + return application + + monkeypatch.setattr(home_eval, "Application", failing_application) + with pytest.raises(RuntimeError, match="Infrastructure failure"): + await home_eval.run(path) + applications[-1].close.assert_awaited_once() + assert (Path(config["test_workspace"]) / "home/metadata/preparation.json").is_file() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("app", ["lme-judge"], indirect=True) +async def test_lme_default_selection_excludes_actual_training_cases(app, inputs, tmp_path): + spec, lme = inputs + original = json.loads(lme.read_text()) + held_out = {**original[0], "question_id": "q2"} + lme.write_text(json.dumps([*original, held_out])) + spec["benchmark"] = {"name": "longmemeval", "source": str(lme), "case_ids": ["q1"]} + _, commit = await prepare_source(app, inputs) + path, config = write_config(tmp_path, app, spec, commit) + config["benchmark"] = {"name": "longmemeval", "source": str(lme), "all_cases": True} + config["exclude_training_cases"] = True + path.write_text(yaml.safe_dump(config)) + result = await home_eval.run(path, prepare_only=True, expected_benchmark="longmemeval") + target = Path(result["test_workspace"]) / "home" + _, dataset, cases = load_inputs(target, EvaluationSpec(record_history=False)) + assert dataset.case_ids == ["q2"] + assert [case.case_id for case in cases] == ["q2"] + assert result["case_count"] == result["query_count"] == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("change", ["candidate", "dataset", "runtime", "test_code", "test_data"]) +async def test_reopen_rejects_changed_identity_or_artifacts(app, inputs, tmp_path, change): + _, commit = await prepare_source(app, inputs) + path, config = write_config(tmp_path, app, inputs[0], commit) + result = await home_eval.run(path, prepare_only=True) + home = Path(result["test_workspace"]) / "home" + if change == "candidate": + config["code_commit"] = "1234567890" + path.write_text(yaml.safe_dump(config)) + elif change == "dataset": + config["benchmark"]["case_ids"] = ["1"] + path.write_text(yaml.safe_dump(config)) + elif change == "runtime": + (home / "code/runtime/reme/__init__.py").write_text("tampered") + elif change == "test_code": + (home / "code/repo/plugins/beam/src/reme_beam/__init__.py").write_text("tampered") + else: + file = next((home / "datasets/training-set/cases").glob("*.json")) + case = read_json(file) + case["queries"][0]["question"] = "tampered" + file.write_text(json.dumps(case)) + before = snapshot(home) + with pytest.raises((ValueError, CalledProcessError)): + await home_eval.run(path, prepare_only=True) + assert snapshot(home) == before + + +@pytest.mark.asyncio +@pytest.mark.parametrize("location", ["same", "child", "parent"]) +async def test_rejects_overlapping_workspaces_before_start(app, inputs, tmp_path, monkeypatch, location): + _, commit = await prepare_source(app, inputs) + path, config = write_config(tmp_path, app, inputs[0], commit) + source = Path(app.config.workspace_dir) + target = {"same": source, "child": source / "test", "parent": source.parent}[location] + config["test_workspace"] = str(target) + path.write_text(yaml.safe_dump(config)) + factory = AsyncMock() + monkeypatch.setattr(home_eval, "Application", factory) + with pytest.raises(ValueError, match="non-nested"): + await home_eval.run(path) + factory.assert_not_called() + + +@pytest.mark.asyncio +async def test_import_runtime_drift_rolls_back_and_allows_retry(app, inputs, tmp_path, monkeypatch): + source_home, commit = await prepare_source(app, inputs) + path, config = write_config(tmp_path, app, inputs[0], commit) + install = EvaluationSource.install + + def fail_install(self, runtime, candidate): + runtime.mkdir() + (runtime / "partial").write_text("partial import") + raise ValueError("Source fixed runtime changed during import") + + monkeypatch.setattr(EvaluationSource, "install", fail_install) + with pytest.raises(ValueError, match="during import"): + await home_eval.run(path, prepare_only=True) + target = Path(config["test_workspace"]) / "home" + assert not (target / "metadata/preparation.json").exists() + assert not list((target / "code/runtime").iterdir()) + assert not list((target / "code/repo").iterdir()) + monkeypatch.setattr(EvaluationSource, "install", install) + await home_eval.run(path, prepare_only=True) + assert tree_fingerprint(target / "code/runtime") == tree_fingerprint(source_home / "code/runtime") + + +@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"]) +@pytest.mark.asyncio +async def test_benchmark_entry_points_select_correct_defaults(benchmark, monkeypatch): + module = import_module(f"benchmark.{benchmark}.run_home_eval") + runner = AsyncMock(return_value={"status": "completed"}) + monkeypatch.setattr(module, "run_evaluation", runner) + assert await module.run(commit="abcdef0") == {"status": "completed"} + runner.assert_awaited_once_with(module.DEFAULT_CONFIG, expected_benchmark=benchmark, commit="abcdef0") + + +@pytest.mark.parametrize("options", [{"sampling": True}, {"record_history": True}, {"max_validations": 1}]) +def test_test_runner_rejects_optimizer_modes(options): + with pytest.raises(ValueError): + home_eval.TestRunConfig.model_validate( + { + "source_workspace": "/source", + "test_workspace": "/test", + "code_commit": "abcdef0", + "benchmark": {"name": "beam", "source": "/data", "variant": "100K", "all_cases": True}, + "evaluation": {"record_history": False, **options}, + }, + ) diff --git a/plugins/home/tests/test_optimizer_compression.py b/plugins/home/tests/test_optimizer_compression.py index eae651b5..d54b6bc1 100644 --- a/plugins/home/tests/test_optimizer_compression.py +++ b/plugins/home/tests/test_optimizer_compression.py @@ -7,6 +7,7 @@ import asyncio import json from agentscope.agent import Agent, ContextConfig +from agentscope.exception import StructuredOutputError from agentscope.message import AssistantMsg, TextBlock, UserMsg, Usage from agentscope.state import AgentState import jsonschema @@ -45,12 +46,19 @@ def make_agent(tmp_path, ratio=0.9, trigger=0.8, failure=None, summary="existing @pytest.mark.asyncio +@pytest.mark.parametrize("failure_type", [RuntimeError, StructuredOutputError]) @pytest.mark.parametrize( ("trigger", "ratio", "expected"), [(0.8, 0.79, False), (0.8, 0.85, False), (0.8, 0.9, True), (0.65, 0.82, False), (0.65, 0.825, True)], ) -async def test_real_compression_failure_uses_dev_meta_emergency_threshold(tmp_path, trigger, ratio, expected): - agent = make_agent(tmp_path, ratio, trigger) +async def test_real_compression_failure_uses_dev_meta_emergency_threshold( + tmp_path, + trigger, + ratio, + expected, + failure_type, +): + agent = make_agent(tmp_path, ratio, trigger, failure=failure_type("Invalid structured output")) context_before, summary_before = agent.state.context, agent.state.summary await agent.compress_context() assert agent.validation_cycle_timer.context_compression_count == int(expected) @@ -63,10 +71,16 @@ async def test_real_compression_failure_uses_dev_meta_emergency_threshold(tmp_pa @pytest.mark.asyncio +@pytest.mark.parametrize("failure_type", [RuntimeError, StructuredOutputError]) @pytest.mark.parametrize("summary", ["existing summary", [TextBlock(text="existing summary")]]) @pytest.mark.parametrize("reserve_ratio", [0.2, 0]) -async def test_offload_retains_full_messages_summary_usage_and_recent_context(tmp_path, summary, reserve_ratio): - agent = make_agent(tmp_path, summary=summary) +async def test_offload_retains_full_messages_summary_usage_and_recent_context( + tmp_path, + summary, + reserve_ratio, + failure_type, +): + agent = make_agent(tmp_path, summary=summary, failure=failure_type("Invalid structured output")) agent.context_config.reserve_ratio = reserve_ratio agent.state.context[-1] = AssistantMsg( id=agent.state.reply_id, @@ -87,7 +101,7 @@ async def test_offload_retains_full_messages_summary_usage_and_recent_context(tm assert data["session_id"] == agent.state.session_id assert data["context_ratio"] == 0.9 assert data["emergency_ratio"] == 0.9 - assert data["failure"]["type"] == "RuntimeError" + assert data["failure"]["type"] == failure_type.__name__ text = agent.state.summary if isinstance(text, list): text = "\n".join(block.text for block in text) @@ -117,6 +131,7 @@ async def test_unrecoverable_failures_and_cancellation_propagate(tmp_path, failu ("failure", "expected"), [ (RuntimeError("provider failed"), True), + (StructuredOutputError("invalid summary"), True), (RuntimeError("The system prompt exceeds threshold"), False), (jsonschema.ValidationError("invalid summary"), True), (openai.OpenAIError("provider unavailable"), True),