From 93898e314b8afd10cb4cebade9120ab9bb7c6fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=85=B1=E7=89=9B=E8=82=89?= Date: Mon, 2 Nov 2026 21:21:48 +0800 Subject: [PATCH] fix(home): load runner credentials and localize dataset evaluation --- benchmark/beam/run_home.py | 2 + benchmark/beam/run_home_eval.py | 211 ++++++++++++++++++- benchmark/home_eval.py | 207 ------------------- benchmark/longmemeval/run_home.py | 2 + benchmark/longmemeval/run_home_eval.py | 213 +++++++++++++++++++- plugins/home/tests/test_benchmark_runner.py | 42 ++++ plugins/home/tests/test_home_eval.py | 71 +++++-- 7 files changed, 513 insertions(+), 235 deletions(-) delete mode 100644 benchmark/home_eval.py diff --git a/benchmark/beam/run_home.py b/benchmark/beam/run_home.py index b231f64d..e70c4359 100644 --- a/benchmark/beam/run_home.py +++ b/benchmark/beam/run_home.py @@ -20,6 +20,7 @@ from reme_home.judge import JUDGE_PLUGINS # noqa: E402 from reme_home.sampling import SamplingPolicy # noqa: E402 from reme.application import Application # noqa: E402 +from reme.utils.env_utils import load_env # noqa: E402 # pylint: enable=wrong-import-position @@ -67,6 +68,7 @@ def load_config(config_path: str | Path, *, workspace_dir: str | None = None, pr async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, instruction=None): """Apply the same runner configuration to each preparation or optimization lifecycle.""" workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir) + load_env(PROJECT_ROOT / ".env", override=False) app = Application( workspace_dir=str(workspace), plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]], diff --git a/benchmark/beam/run_home_eval.py b/benchmark/beam/run_home_eval.py index bb028e94..13513d37 100644 --- a/benchmark/beam/run_home_eval.py +++ b/benchmark/beam/run_home_eval.py @@ -1,21 +1,226 @@ """Evaluate a BEAM Home candidate in an independent test workspace.""" +import argparse +import asyncio +import json import sys from pathlib import Path +from pydantic import Field, model_validator +import yaml + +# Support direct execution from any working directory as well as module imports. PROJECT_ROOT = Path(__file__).resolve().parents[2] if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from benchmark.home_eval import main, run as run_evaluation # noqa: E402 # pylint: disable=wrong-import-position +# pylint: disable=wrong-import-position +from reme_home.code.repository import resolve_revision, validate_repository # noqa: E402 +from reme_home.evaluation.artifacts import contained, read_json # noqa: E402 +from reme_home.evaluation.component import blocking # noqa: E402 +from reme_home.evaluation.inputs import EvaluationSpec, load_inputs # noqa: E402 +from reme_home.judge import JUDGE_PLUGINS # noqa: E402 +from reme_home.preparation.code import tree_fingerprint # noqa: E402 +from reme_home.preparation.data.models import fingerprint # noqa: E402 +from reme_home.preparation.evaluation_source import EvaluationSource # noqa: E402 +from reme_home.preparation.spec import BenchmarkSpec, DebugSpec, SpecModel # noqa: E402 + +from reme.application import Application # noqa: E402 +from reme.utils.env_utils import load_env # noqa: E402 + +# pylint: enable=wrong-import-position DEFAULT_CONFIG = Path(__file__).with_name("config_home_eval.yaml") +class TestRunConfig(SpecModel): + """Explicit dataset selection and execution options for one pinned candidate.""" + + source_workspace: str = Field(min_length=1) + test_workspace: str = Field(min_length=1) + code_commit: str = Field(min_length=4) + benchmark: BenchmarkSpec + exclude_training_cases: bool = False + evaluation: EvaluationSpec = Field(default_factory=lambda: EvaluationSpec(record_history=False)) + + @model_validator(mode="after") + def validate_test_mode(self): + """Test runs score every prepared query and never consume optimizer budgets.""" + options = self.evaluation + if any( + ( + options.debug, + options.sampling, + options.record_history, + options.max_evaluations is not None, + options.sampling_policy is not None, + options.case_ids is not None, + options.query_ids is not None, + options.reuse_construction_from is not None, + ), + ): + raise ValueError( + "Test evaluation requires full scoring with record_history=false; select cases in benchmark", + ) + if self.exclude_training_cases and (self.benchmark.name != "longmemeval" or not self.benchmark.all_cases): + raise ValueError("exclude_training_cases requires LongMemEval with all_cases=true") + return self + + +def project_path(value: str, project_root: Path) -> Path: + """Resolve runner paths independently of the launching working directory.""" + if value.startswith("~"): + raise ValueError("Use absolute or project-relative paths, not home-relative paths") + return (project_root / value).resolve() + + +def load_request(config_path, *, commit=None, source_workspace=None, test_workspace=None, project_root=PROJECT_ROOT): + """Validate all source inputs before starting an application or writing test data.""" + raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8")) + for key, value in ( + ("code_commit", commit), + ("source_workspace", source_workspace), + ("test_workspace", test_workspace), + ): + if value is not None: + raw[key] = value + # Execution settings can be partial while retaining the test-only history default. + raw["evaluation"] = {"record_history": False, **raw.get("evaluation", {})} + config = TestRunConfig.model_validate(raw) + source_path = project_path(config.source_workspace, project_root) + target = project_path(config.test_workspace, project_root) + if target.is_relative_to(source_path) or source_path.is_relative_to(target): + raise ValueError("Test and source workspaces must be separate, non-nested directories") + if target.exists() and ( + not target.is_dir() or (any(target.iterdir()) and not (target / "home/metadata/workspace.json").is_file()) + ): + raise ValueError("Use an empty test workspace or a compatible prepared Home test workspace") + source = EvaluationSource.load(source_path, config.code_commit) + benchmark = config.benchmark.model_copy(deep=True) + if benchmark.name != source.spec.benchmark.name: + raise ValueError("Test benchmark must match the source experiment") + benchmark.source = str(project_path(benchmark.source, project_root).resolve(strict=True)) + if benchmark.evidence_source is not None: + benchmark.evidence_source = str(project_path(benchmark.evidence_source, project_root).resolve(strict=True)) + if config.exclude_training_cases: + items = json.loads(Path(benchmark.source).read_text(encoding="utf-8")) + ids = [str(item["question_id"]) for item in items] + if len(ids) != len(set(ids)): + raise ValueError("Duplicate LongMemEval question IDs") + training = set(source.record["dataset"]["case_ids"]) + if training - set(ids): + raise ValueError("Test dataset is missing training IDs; provide the original full dataset") + benchmark.case_ids = [case_id for case_id in ids if case_id not in training] + benchmark.all_cases = False + benchmark = BenchmarkSpec.model_validate(benchmark.model_dump()) + spec = source.spec.model_copy(deep=True) + spec.benchmark = benchmark + spec.debug = DebugSpec(enabled=False) + return config, target, source, spec + + +def existing_preparation(home, source, spec): + """Reopen only the same request with unchanged code, runtime and normalized data.""" + marker = contained(home, "metadata/preparation.json") + if not marker.exists(): + return None + record = read_json(marker) + expected = {**source.provenance, "request_fingerprint": fingerprint(spec.model_dump(mode="json"))} + if record.get("test_evaluation") != expected: + raise ValueError("Test workspace belongs to a different candidate or dataset; use a new workspace") + domain = yaml.safe_load(contained(home, "domain_spec.yaml").read_text(encoding="utf-8")) + if fingerprint(domain) != record["config_fingerprint"]: + raise ValueError("Test preparation configuration has changed") + repository, _ = validate_repository(home, record) + if ( + resolve_revision(repository, "HEAD") != record["code"]["initial_commit"] + or tree_fingerprint(repository) != record["code"]["candidate_fingerprint"] + or tree_fingerprint(contained(home, "code/runtime", directory=True)) != source.provenance["runtime_fingerprint"] + ): + raise ValueError("Prepared test code or runtime has changed") + load_inputs(home, EvaluationSpec(record_history=False)) + return record + + +async def run_evaluation( + config_path, + *, + commit=None, + source_workspace=None, + test_workspace=None, + prepare_only=False, + expected_benchmark=None, +): + """Prepare transactionally, then score through the normal application lifecycle.""" + config, target, source, spec = await blocking( + load_request, + config_path, + commit=commit, + source_workspace=source_workspace, + test_workspace=test_workspace, + ) + if expected_benchmark is not None and config.benchmark.name != expected_benchmark: + raise ValueError(f"This entry point requires benchmark.name={expected_benchmark}") + load_env(PROJECT_ROOT / ".env", override=False) + app = Application( + workspace_dir=str(target), + plugins=["home", JUDGE_PLUGINS[config.benchmark.name]], + service={"backend": "cli"}, + enable_logo=False, + log_to_file=False, + ) + try: + await app.start() + home = target / "home" + prepared = await blocking(existing_preparation, home, source, spec) + if prepared is None: + preparation = app.context.components["home.preparation"]["default"] + prepared = await preparation.prepare(spec=spec.model_dump(mode="json"), code_source=source) + result = { + "test_workspace": str(target), + **source.provenance, + "evaluation_commit_sha": prepared["code"]["initial_commit"], + "case_count": prepared["dataset"]["case_count"], + "query_count": prepared["dataset"]["query_count"], + } + if not prepare_only: + response = await app.run_job( + "home_evaluate", + **config.evaluation.model_dump(mode="json", exclude_unset=True), + ) + if not response.success: + raise RuntimeError(response.answer) + result["evaluation"] = response.answer + return result + finally: + await app.close() + + async def run(config_path=DEFAULT_CONFIG, **kwargs): - """Score the selected BEAM commit without changing its optimization workspace.""" + """Score the selected commit in its own Home workspace.""" return await run_evaluation(config_path, expected_benchmark="beam", **kwargs) +def main(): + """Explicit local one-shot entry point, with preparation-only service setup.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + parser.add_argument("--commit", help="Candidate commit SHA or unambiguous hexadecimal prefix") + parser.add_argument("--source-workspace", help="Prepared optimization workspace") + parser.add_argument("--test-workspace", help="Independent test output workspace") + parser.add_argument("--prepare-only", action="store_true", help="Prepare for a later home_evaluate service call") + args = parser.parse_args() + result = asyncio.run( + run( + args.config, + commit=args.commit, + source_workspace=args.source_workspace, + test_workspace=args.test_workspace, + prepare_only=args.prepare_only, + ), + ) + print(json.dumps(result, ensure_ascii=False, indent=2)) + + if __name__ == "__main__": - main("beam", DEFAULT_CONFIG) + main() diff --git a/benchmark/home_eval.py b/benchmark/home_eval.py deleted file mode 100644 index 427ace0c..00000000 --- a/benchmark/home_eval.py +++ /dev/null @@ -1,207 +0,0 @@ -"""Standalone test evaluation shared by the BEAM and LongMemEval entry points.""" - -import argparse -import asyncio -import json -from pathlib import Path - -from pydantic import Field, model_validator -import yaml - -from reme_home.code.repository import resolve_revision, validate_repository -from reme_home.evaluation.artifacts import contained, read_json -from reme_home.evaluation.component import blocking -from reme_home.evaluation.inputs import EvaluationSpec, load_inputs -from reme_home.judge import JUDGE_PLUGINS -from reme_home.preparation.code import tree_fingerprint -from reme_home.preparation.data.models import fingerprint -from reme_home.preparation.evaluation_source import EvaluationSource -from reme_home.preparation.spec import BenchmarkSpec, DebugSpec, SpecModel - -from reme.application import Application - -PROJECT_ROOT = Path(__file__).resolve().parents[1] - - -class TestRunConfig(SpecModel): - """Explicit dataset selection and execution options for one pinned candidate.""" - - source_workspace: str = Field(min_length=1) - test_workspace: str = Field(min_length=1) - code_commit: str = Field(min_length=4) - benchmark: BenchmarkSpec - exclude_training_cases: bool = False - evaluation: EvaluationSpec = Field(default_factory=lambda: EvaluationSpec(record_history=False)) - - @model_validator(mode="after") - def validate_test_mode(self): - """Test runs score every prepared query and never consume optimizer budgets.""" - options = self.evaluation - if any( - ( - options.debug, - options.sampling, - options.record_history, - options.max_evaluations is not None, - options.sampling_policy is not None, - options.case_ids is not None, - options.query_ids is not None, - options.reuse_construction_from is not None, - ), - ): - raise ValueError( - "Test evaluation requires full scoring with record_history=false; select cases in benchmark", - ) - if self.exclude_training_cases and (self.benchmark.name != "longmemeval" or not self.benchmark.all_cases): - raise ValueError("exclude_training_cases requires LongMemEval with all_cases=true") - return self - - -def project_path(value: str, project_root: Path) -> Path: - """Resolve runner paths independently of the launching working directory.""" - if value.startswith("~"): - raise ValueError("Use absolute or project-relative paths, not home-relative paths") - return (project_root / value).resolve() - - -def load_request(config_path, *, commit=None, source_workspace=None, test_workspace=None, project_root=PROJECT_ROOT): - """Validate all source inputs before starting an application or writing test data.""" - raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8")) - for key, value in ( - ("code_commit", commit), - ("source_workspace", source_workspace), - ("test_workspace", test_workspace), - ): - if value is not None: - raw[key] = value - # Execution settings can be partial while retaining the test-only history default. - raw["evaluation"] = {"record_history": False, **raw.get("evaluation", {})} - config = TestRunConfig.model_validate(raw) - source_path = project_path(config.source_workspace, project_root) - target = project_path(config.test_workspace, project_root) - if target.is_relative_to(source_path) or source_path.is_relative_to(target): - raise ValueError("Test and source workspaces must be separate, non-nested directories") - if target.exists() and ( - not target.is_dir() or (any(target.iterdir()) and not (target / "home/metadata/workspace.json").is_file()) - ): - raise ValueError("Use an empty test workspace or a compatible prepared Home test workspace") - source = EvaluationSource.load(source_path, config.code_commit) - benchmark = config.benchmark.model_copy(deep=True) - if benchmark.name != source.spec.benchmark.name: - raise ValueError("Test benchmark must match the source experiment") - benchmark.source = str(project_path(benchmark.source, project_root).resolve(strict=True)) - if benchmark.evidence_source is not None: - benchmark.evidence_source = str(project_path(benchmark.evidence_source, project_root).resolve(strict=True)) - if config.exclude_training_cases: - items = json.loads(Path(benchmark.source).read_text(encoding="utf-8")) - ids = [str(item["question_id"]) for item in items] - if len(ids) != len(set(ids)): - raise ValueError("Duplicate LongMemEval question IDs") - training = set(source.record["dataset"]["case_ids"]) - if training - set(ids): - raise ValueError("Test dataset is missing training IDs; provide the original full dataset") - benchmark.case_ids = [case_id for case_id in ids if case_id not in training] - benchmark.all_cases = False - benchmark = BenchmarkSpec.model_validate(benchmark.model_dump()) - spec = source.spec.model_copy(deep=True) - spec.benchmark = benchmark - spec.debug = DebugSpec(enabled=False) - return config, target, source, spec - - -def existing_preparation(home, source, spec): - """Reopen only the same request with unchanged code, runtime and normalized data.""" - marker = contained(home, "metadata/preparation.json") - if not marker.exists(): - return None - record = read_json(marker) - expected = {**source.provenance, "request_fingerprint": fingerprint(spec.model_dump(mode="json"))} - if record.get("test_evaluation") != expected: - raise ValueError("Test workspace belongs to a different candidate or dataset; use a new workspace") - domain = yaml.safe_load(contained(home, "domain_spec.yaml").read_text(encoding="utf-8")) - if fingerprint(domain) != record["config_fingerprint"]: - raise ValueError("Test preparation configuration has changed") - repository, _ = validate_repository(home, record) - if ( - resolve_revision(repository, "HEAD") != record["code"]["initial_commit"] - or tree_fingerprint(repository) != record["code"]["candidate_fingerprint"] - or tree_fingerprint(contained(home, "code/runtime", directory=True)) != source.provenance["runtime_fingerprint"] - ): - raise ValueError("Prepared test code or runtime has changed") - load_inputs(home, EvaluationSpec(record_history=False)) - return record - - -async def run( - config_path, - *, - commit=None, - source_workspace=None, - test_workspace=None, - prepare_only=False, - expected_benchmark=None, -): - """Prepare transactionally, then score through the normal application lifecycle.""" - config, target, source, spec = await blocking( - load_request, - config_path, - commit=commit, - source_workspace=source_workspace, - test_workspace=test_workspace, - ) - if expected_benchmark is not None and config.benchmark.name != expected_benchmark: - raise ValueError(f"This entry point requires benchmark.name={expected_benchmark}") - app = Application( - workspace_dir=str(target), - plugins=["home", JUDGE_PLUGINS[config.benchmark.name]], - service={"backend": "cli"}, - enable_logo=False, - log_to_file=False, - ) - try: - await app.start() - home = target / "home" - prepared = await blocking(existing_preparation, home, source, spec) - if prepared is None: - preparation = app.context.components["home.preparation"]["default"] - prepared = await preparation.prepare(spec=spec.model_dump(mode="json"), code_source=source) - result = { - "test_workspace": str(target), - **source.provenance, - "evaluation_commit_sha": prepared["code"]["initial_commit"], - "case_count": prepared["dataset"]["case_count"], - "query_count": prepared["dataset"]["query_count"], - } - if not prepare_only: - response = await app.run_job( - "home_evaluate", - **config.evaluation.model_dump(mode="json", exclude_unset=True), - ) - if not response.success: - raise RuntimeError(response.answer) - result["evaluation"] = response.answer - return result - finally: - await app.close() - - -def main(benchmark, default_config): - """Explicit local one-shot entry point, with preparation-only service setup.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--config", type=Path, default=default_config) - parser.add_argument("--commit", help="Candidate commit SHA or unambiguous hexadecimal prefix") - parser.add_argument("--source-workspace", help="Prepared optimization workspace") - parser.add_argument("--test-workspace", help="Independent test output workspace") - parser.add_argument("--prepare-only", action="store_true", help="Prepare for a later home_evaluate service call") - args = parser.parse_args() - result = asyncio.run( - run( - args.config, - commit=args.commit, - source_workspace=args.source_workspace, - test_workspace=args.test_workspace, - prepare_only=args.prepare_only, - expected_benchmark=benchmark, - ), - ) - print(json.dumps(result, ensure_ascii=False, indent=2)) diff --git a/benchmark/longmemeval/run_home.py b/benchmark/longmemeval/run_home.py index f29059e9..1ab63cd1 100644 --- a/benchmark/longmemeval/run_home.py +++ b/benchmark/longmemeval/run_home.py @@ -20,6 +20,7 @@ from reme_home.judge import JUDGE_PLUGINS # noqa: E402 from reme_home.sampling import SamplingPolicy # noqa: E402 from reme.application import Application # noqa: E402 +from reme.utils.env_utils import load_env # noqa: E402 # pylint: enable=wrong-import-position @@ -67,6 +68,7 @@ def load_config(config_path: str | Path, *, workspace_dir: str | None = None, pr async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, instruction=None): """Apply the same runner configuration to each preparation or optimization lifecycle.""" workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir) + load_env(PROJECT_ROOT / ".env", override=False) app = Application( workspace_dir=str(workspace), plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]], diff --git a/benchmark/longmemeval/run_home_eval.py b/benchmark/longmemeval/run_home_eval.py index bb5ec3ee..93d36f27 100644 --- a/benchmark/longmemeval/run_home_eval.py +++ b/benchmark/longmemeval/run_home_eval.py @@ -1,21 +1,226 @@ -"""Evaluate a LongMemEval Home candidate on an independent test dataset.""" +"""Evaluate a LongMemEval Home candidate in an independent test workspace.""" +import argparse +import asyncio +import json import sys from pathlib import Path +from pydantic import Field, model_validator +import yaml + +# Support direct execution from any working directory as well as module imports. PROJECT_ROOT = Path(__file__).resolve().parents[2] if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from benchmark.home_eval import main, run as run_evaluation # noqa: E402 # pylint: disable=wrong-import-position +# pylint: disable=wrong-import-position +from reme_home.code.repository import resolve_revision, validate_repository # noqa: E402 +from reme_home.evaluation.artifacts import contained, read_json # noqa: E402 +from reme_home.evaluation.component import blocking # noqa: E402 +from reme_home.evaluation.inputs import EvaluationSpec, load_inputs # noqa: E402 +from reme_home.judge import JUDGE_PLUGINS # noqa: E402 +from reme_home.preparation.code import tree_fingerprint # noqa: E402 +from reme_home.preparation.data.models import fingerprint # noqa: E402 +from reme_home.preparation.evaluation_source import EvaluationSource # noqa: E402 +from reme_home.preparation.spec import BenchmarkSpec, DebugSpec, SpecModel # noqa: E402 + +from reme.application import Application # noqa: E402 +from reme.utils.env_utils import load_env # noqa: E402 + +# pylint: enable=wrong-import-position DEFAULT_CONFIG = Path(__file__).with_name("config_home_eval.yaml") +class TestRunConfig(SpecModel): + """Explicit dataset selection and execution options for one pinned candidate.""" + + source_workspace: str = Field(min_length=1) + test_workspace: str = Field(min_length=1) + code_commit: str = Field(min_length=4) + benchmark: BenchmarkSpec + exclude_training_cases: bool = False + evaluation: EvaluationSpec = Field(default_factory=lambda: EvaluationSpec(record_history=False)) + + @model_validator(mode="after") + def validate_test_mode(self): + """Test runs score every prepared query and never consume optimizer budgets.""" + options = self.evaluation + if any( + ( + options.debug, + options.sampling, + options.record_history, + options.max_evaluations is not None, + options.sampling_policy is not None, + options.case_ids is not None, + options.query_ids is not None, + options.reuse_construction_from is not None, + ), + ): + raise ValueError( + "Test evaluation requires full scoring with record_history=false; select cases in benchmark", + ) + if self.exclude_training_cases and (self.benchmark.name != "longmemeval" or not self.benchmark.all_cases): + raise ValueError("exclude_training_cases requires LongMemEval with all_cases=true") + return self + + +def project_path(value: str, project_root: Path) -> Path: + """Resolve runner paths independently of the launching working directory.""" + if value.startswith("~"): + raise ValueError("Use absolute or project-relative paths, not home-relative paths") + return (project_root / value).resolve() + + +def load_request(config_path, *, commit=None, source_workspace=None, test_workspace=None, project_root=PROJECT_ROOT): + """Validate all source inputs before starting an application or writing test data.""" + raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8")) + for key, value in ( + ("code_commit", commit), + ("source_workspace", source_workspace), + ("test_workspace", test_workspace), + ): + if value is not None: + raw[key] = value + # Execution settings can be partial while retaining the test-only history default. + raw["evaluation"] = {"record_history": False, **raw.get("evaluation", {})} + config = TestRunConfig.model_validate(raw) + source_path = project_path(config.source_workspace, project_root) + target = project_path(config.test_workspace, project_root) + if target.is_relative_to(source_path) or source_path.is_relative_to(target): + raise ValueError("Test and source workspaces must be separate, non-nested directories") + if target.exists() and ( + not target.is_dir() or (any(target.iterdir()) and not (target / "home/metadata/workspace.json").is_file()) + ): + raise ValueError("Use an empty test workspace or a compatible prepared Home test workspace") + source = EvaluationSource.load(source_path, config.code_commit) + benchmark = config.benchmark.model_copy(deep=True) + if benchmark.name != source.spec.benchmark.name: + raise ValueError("Test benchmark must match the source experiment") + benchmark.source = str(project_path(benchmark.source, project_root).resolve(strict=True)) + if benchmark.evidence_source is not None: + benchmark.evidence_source = str(project_path(benchmark.evidence_source, project_root).resolve(strict=True)) + if config.exclude_training_cases: + items = json.loads(Path(benchmark.source).read_text(encoding="utf-8")) + ids = [str(item["question_id"]) for item in items] + if len(ids) != len(set(ids)): + raise ValueError("Duplicate LongMemEval question IDs") + training = set(source.record["dataset"]["case_ids"]) + if training - set(ids): + raise ValueError("Test dataset is missing training IDs; provide the original full dataset") + benchmark.case_ids = [case_id for case_id in ids if case_id not in training] + benchmark.all_cases = False + benchmark = BenchmarkSpec.model_validate(benchmark.model_dump()) + spec = source.spec.model_copy(deep=True) + spec.benchmark = benchmark + spec.debug = DebugSpec(enabled=False) + return config, target, source, spec + + +def existing_preparation(home, source, spec): + """Reopen only the same request with unchanged code, runtime and normalized data.""" + marker = contained(home, "metadata/preparation.json") + if not marker.exists(): + return None + record = read_json(marker) + expected = {**source.provenance, "request_fingerprint": fingerprint(spec.model_dump(mode="json"))} + if record.get("test_evaluation") != expected: + raise ValueError("Test workspace belongs to a different candidate or dataset; use a new workspace") + domain = yaml.safe_load(contained(home, "domain_spec.yaml").read_text(encoding="utf-8")) + if fingerprint(domain) != record["config_fingerprint"]: + raise ValueError("Test preparation configuration has changed") + repository, _ = validate_repository(home, record) + if ( + resolve_revision(repository, "HEAD") != record["code"]["initial_commit"] + or tree_fingerprint(repository) != record["code"]["candidate_fingerprint"] + or tree_fingerprint(contained(home, "code/runtime", directory=True)) != source.provenance["runtime_fingerprint"] + ): + raise ValueError("Prepared test code or runtime has changed") + load_inputs(home, EvaluationSpec(record_history=False)) + return record + + +async def run_evaluation( + config_path, + *, + commit=None, + source_workspace=None, + test_workspace=None, + prepare_only=False, + expected_benchmark=None, +): + """Prepare transactionally, then score through the normal application lifecycle.""" + config, target, source, spec = await blocking( + load_request, + config_path, + commit=commit, + source_workspace=source_workspace, + test_workspace=test_workspace, + ) + if expected_benchmark is not None and config.benchmark.name != expected_benchmark: + raise ValueError(f"This entry point requires benchmark.name={expected_benchmark}") + load_env(PROJECT_ROOT / ".env", override=False) + app = Application( + workspace_dir=str(target), + plugins=["home", JUDGE_PLUGINS[config.benchmark.name]], + service={"backend": "cli"}, + enable_logo=False, + log_to_file=False, + ) + try: + await app.start() + home = target / "home" + prepared = await blocking(existing_preparation, home, source, spec) + if prepared is None: + preparation = app.context.components["home.preparation"]["default"] + prepared = await preparation.prepare(spec=spec.model_dump(mode="json"), code_source=source) + result = { + "test_workspace": str(target), + **source.provenance, + "evaluation_commit_sha": prepared["code"]["initial_commit"], + "case_count": prepared["dataset"]["case_count"], + "query_count": prepared["dataset"]["query_count"], + } + if not prepare_only: + response = await app.run_job( + "home_evaluate", + **config.evaluation.model_dump(mode="json", exclude_unset=True), + ) + if not response.success: + raise RuntimeError(response.answer) + result["evaluation"] = response.answer + return result + finally: + await app.close() + + async def run(config_path=DEFAULT_CONFIG, **kwargs): - """Score the selected LongMemEval commit in its own Home workspace.""" + """Score the selected commit in its own Home workspace.""" return await run_evaluation(config_path, expected_benchmark="longmemeval", **kwargs) +def main(): + """Explicit local one-shot entry point, with preparation-only service setup.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + parser.add_argument("--commit", help="Candidate commit SHA or unambiguous hexadecimal prefix") + parser.add_argument("--source-workspace", help="Prepared optimization workspace") + parser.add_argument("--test-workspace", help="Independent test output workspace") + parser.add_argument("--prepare-only", action="store_true", help="Prepare for a later home_evaluate service call") + args = parser.parse_args() + result = asyncio.run( + run( + args.config, + commit=args.commit, + source_workspace=args.source_workspace, + test_workspace=args.test_workspace, + prepare_only=args.prepare_only, + ), + ) + print(json.dumps(result, ensure_ascii=False, indent=2)) + + if __name__ == "__main__": - main("longmemeval", DEFAULT_CONFIG) + main() diff --git a/plugins/home/tests/test_benchmark_runner.py b/plugins/home/tests/test_benchmark_runner.py index 46e6f359..68fc471c 100644 --- a/plugins/home/tests/test_benchmark_runner.py +++ b/plugins/home/tests/test_benchmark_runner.py @@ -2,9 +2,11 @@ import hashlib import json +import os import sys from importlib import import_module from pathlib import Path +from types import SimpleNamespace from unittest.mock import AsyncMock import pytest @@ -18,6 +20,46 @@ from benchmark.longmemeval import run_home as lme_runner ROOT = Path(__file__).resolve().parents[3] +@pytest.fixture(autouse=True) +def isolated_runner_environment(tmp_path, monkeypatch): + """Never load developer credentials while exercising benchmark lifecycles.""" + for benchmark in ("beam", "longmemeval"): + for runner in ("run_home", "run_home_eval"): + module = import_module(f"benchmark.{benchmark}.{runner}") + monkeypatch.setattr(module, "PROJECT_ROOT", tmp_path) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"]) +@pytest.mark.parametrize("runner", ["run_home", "run_home_eval"]) +@pytest.mark.parametrize("env_file", [True, False]) +async def test_credentials_loaded_before_application(tmp_path, monkeypatch, benchmark, runner, env_file): + """Load repository .env without overriding exports, independently of cwd.""" + module = import_module(f"benchmark.{benchmark}.{runner}") + monkeypatch.delenv("EMBEDDING_API_KEY", raising=False) + monkeypatch.setenv("LLM_API_KEY", "exported-key") + if env_file: + (tmp_path / ".env").write_text("LLM_API_KEY=file-key\nEMBEDDING_API_KEY=embedding-key\n") + cwd = tmp_path / "elsewhere" + cwd.mkdir() + monkeypatch.chdir(cwd) + if runner == "run_home_eval": + config = SimpleNamespace(benchmark=SimpleNamespace(name=benchmark)) + monkeypatch.setattr(module, "load_request", lambda *args, **kwargs: (config, tmp_path, None, None)) + + class ApplicationReached(Exception): + """Stop before creating any application resources.""" + + def factory(**_kwargs): + assert os.environ["LLM_API_KEY"] == "exported-key" + assert os.environ.get("EMBEDDING_API_KEY") == ("embedding-key" if env_file else None) + raise ApplicationReached + + monkeypatch.setattr(module, "Application", factory) + with pytest.raises(ApplicationReached): + await module.run() + + def test_preserved_lme_split_and_evidence(): """Retain dev_meta's exact reviewed partition and annotations.""" directory = ROOT / "benchmark/longmemeval" diff --git a/plugins/home/tests/test_home_eval.py b/plugins/home/tests/test_home_eval.py index 29092e5d..033b106d 100644 --- a/plugins/home/tests/test_home_eval.py +++ b/plugins/home/tests/test_home_eval.py @@ -27,11 +27,17 @@ from test_preparation import ( # noqa: F401 # pylint: disable=unused-import snapshot, ) -from benchmark import home_eval from reme.schema import Response from reme.components.agent_wrapper import AsAgentWrapper +@pytest.fixture(params=["beam", "longmemeval"]) +def home_eval(request, tmp_path, monkeypatch): + module = import_module(f"benchmark.{request.param}.run_home_eval") + monkeypatch.setattr(module, "PROJECT_ROOT", tmp_path) + return module + + def write_config(tmp_path, app, spec, commit): config = { "source_workspace": str(app.config.workspace_dir), @@ -54,7 +60,12 @@ async def prepare_source(app, inputs): @pytest.mark.asyncio -async def test_import_historical_commit_ignores_dirty_checkout_and_missing_original_repo(app, inputs, tmp_path): +async def test_import_historical_commit_ignores_dirty_checkout_and_missing_original_repo( + home_eval, + app, + inputs, + tmp_path, +): home, _ = await prepare_source(app, inputs) repo = home / "code/repo" command(repo, "switch", "-c", "candidate") @@ -72,7 +83,7 @@ async def test_import_historical_commit_ignores_dirty_checkout_and_missing_origi path, _ = write_config(tmp_path, app, spec, selected[:10]) before = snapshot(home) - result = await home_eval.run(path, prepare_only=True) + result = await home_eval.run_evaluation(path, prepare_only=True) assert result["source_commit_sha"] == selected target = Path(result["test_workspace"]) / "home" @@ -88,10 +99,16 @@ async def test_import_historical_commit_ignores_dirty_checkout_and_missing_origi @pytest.mark.asyncio -async def test_reopen_scores_all_queries_without_optimizer_history_and_closes(app, inputs, tmp_path, monkeypatch): +async def test_reopen_scores_all_queries_without_optimizer_history_and_closes( + home_eval, + app, + inputs, + tmp_path, + monkeypatch, +): home, commit = await prepare_source(app, inputs) path, _ = write_config(tmp_path, app, inputs[0], commit) - await home_eval.run(path, prepare_only=True) + await home_eval.run_evaluation(path, prepare_only=True) applications = [] factory = home_eval.Application @@ -114,7 +131,7 @@ async def test_reopen_scores_all_queries_without_optimizer_history_and_closes(ap return application monkeypatch.setattr(home_eval, "Application", tracked_application) - result = await home_eval.run(path) + result = await home_eval.run_evaluation(path) assert result["evaluation"] == {"result_dir": "test-results"} applications[-1].close.assert_awaited_once() assert not list((home / "evaluations").rglob("manifest.json")) @@ -123,7 +140,13 @@ async def test_reopen_scores_all_queries_without_optimizer_history_and_closes(ap @pytest.mark.asyncio @pytest.mark.parametrize("app", ["beam-judge", "lme-judge"], indirect=True) -async def test_full_evaluation_records_source_provenance_with_fake_docker(app, inputs, tmp_path, monkeypatch): +async def test_full_evaluation_records_source_provenance_with_fake_docker( + home_eval, + app, + inputs, + tmp_path, + monkeypatch, +): spec, lme = inputs if "lme-judge" in app.config.plugins: spec["benchmark"] = {"name": "longmemeval", "source": str(lme), "all_cases": True} @@ -138,7 +161,7 @@ async def test_full_evaluation_records_source_provenance_with_fake_docker(app, i answer = "yes" if spec["benchmark"]["name"] == "longmemeval" else '{"score": 0.5}' monkeypatch.setattr(AsAgentWrapper, "reply", AsyncMock(return_value={"result": answer})) before = snapshot(home) - result = await home_eval.run(path) + result = await home_eval.run_evaluation(path) run = Path(result["evaluation"]["result_dir"]) manifest = read_json(run / "manifest.json") assert manifest["test_evaluation"]["source_commit_sha"] == commit @@ -151,7 +174,13 @@ async def test_full_evaluation_records_source_provenance_with_fake_docker(app, i @pytest.mark.asyncio -async def test_evaluation_failure_closes_application_and_preserves_preparation(app, inputs, tmp_path, monkeypatch): +async def test_evaluation_failure_closes_application_and_preserves_preparation( + home_eval, + app, + inputs, + tmp_path, + monkeypatch, +): _, commit = await prepare_source(app, inputs) path, config = write_config(tmp_path, app, inputs[0], commit) factory = home_eval.Application @@ -166,14 +195,14 @@ async def test_evaluation_failure_closes_application_and_preserves_preparation(a monkeypatch.setattr(home_eval, "Application", failing_application) with pytest.raises(RuntimeError, match="Infrastructure failure"): - await home_eval.run(path) + await home_eval.run_evaluation(path) applications[-1].close.assert_awaited_once() assert (Path(config["test_workspace"]) / "home/metadata/preparation.json").is_file() @pytest.mark.asyncio @pytest.mark.parametrize("app", ["lme-judge"], indirect=True) -async def test_lme_default_selection_excludes_actual_training_cases(app, inputs, tmp_path): +async def test_lme_default_selection_excludes_actual_training_cases(home_eval, app, inputs, tmp_path): spec, lme = inputs original = json.loads(lme.read_text()) held_out = {**original[0], "question_id": "q2"} @@ -184,7 +213,7 @@ async def test_lme_default_selection_excludes_actual_training_cases(app, inputs, config["benchmark"] = {"name": "longmemeval", "source": str(lme), "all_cases": True} config["exclude_training_cases"] = True path.write_text(yaml.safe_dump(config)) - result = await home_eval.run(path, prepare_only=True, expected_benchmark="longmemeval") + result = await home_eval.run_evaluation(path, prepare_only=True, expected_benchmark="longmemeval") target = Path(result["test_workspace"]) / "home" _, dataset, cases = load_inputs(target, EvaluationSpec(record_history=False)) assert dataset.case_ids == ["q2"] @@ -194,10 +223,10 @@ async def test_lme_default_selection_excludes_actual_training_cases(app, inputs, @pytest.mark.asyncio @pytest.mark.parametrize("change", ["candidate", "dataset", "runtime", "test_code", "test_data"]) -async def test_reopen_rejects_changed_identity_or_artifacts(app, inputs, tmp_path, change): +async def test_reopen_rejects_changed_identity_or_artifacts(home_eval, app, inputs, tmp_path, change): _, commit = await prepare_source(app, inputs) path, config = write_config(tmp_path, app, inputs[0], commit) - result = await home_eval.run(path, prepare_only=True) + result = await home_eval.run_evaluation(path, prepare_only=True) home = Path(result["test_workspace"]) / "home" if change == "candidate": config["code_commit"] = "1234567890" @@ -216,13 +245,13 @@ async def test_reopen_rejects_changed_identity_or_artifacts(app, inputs, tmp_pat file.write_text(json.dumps(case)) before = snapshot(home) with pytest.raises((ValueError, CalledProcessError)): - await home_eval.run(path, prepare_only=True) + await home_eval.run_evaluation(path, prepare_only=True) assert snapshot(home) == before @pytest.mark.asyncio @pytest.mark.parametrize("location", ["same", "child", "parent"]) -async def test_rejects_overlapping_workspaces_before_start(app, inputs, tmp_path, monkeypatch, location): +async def test_rejects_overlapping_workspaces_before_start(home_eval, app, inputs, tmp_path, monkeypatch, location): _, commit = await prepare_source(app, inputs) path, config = write_config(tmp_path, app, inputs[0], commit) source = Path(app.config.workspace_dir) @@ -232,12 +261,12 @@ async def test_rejects_overlapping_workspaces_before_start(app, inputs, tmp_path factory = AsyncMock() monkeypatch.setattr(home_eval, "Application", factory) with pytest.raises(ValueError, match="non-nested"): - await home_eval.run(path) + await home_eval.run_evaluation(path) factory.assert_not_called() @pytest.mark.asyncio -async def test_import_runtime_drift_rolls_back_and_allows_retry(app, inputs, tmp_path, monkeypatch): +async def test_import_runtime_drift_rolls_back_and_allows_retry(home_eval, app, inputs, tmp_path, monkeypatch): source_home, commit = await prepare_source(app, inputs) path, config = write_config(tmp_path, app, inputs[0], commit) install = EvaluationSource.install @@ -249,13 +278,13 @@ async def test_import_runtime_drift_rolls_back_and_allows_retry(app, inputs, tmp monkeypatch.setattr(EvaluationSource, "install", fail_install) with pytest.raises(ValueError, match="during import"): - await home_eval.run(path, prepare_only=True) + await home_eval.run_evaluation(path, prepare_only=True) target = Path(config["test_workspace"]) / "home" assert not (target / "metadata/preparation.json").exists() assert not list((target / "code/runtime").iterdir()) assert not list((target / "code/repo").iterdir()) monkeypatch.setattr(EvaluationSource, "install", install) - await home_eval.run(path, prepare_only=True) + await home_eval.run_evaluation(path, prepare_only=True) assert tree_fingerprint(target / "code/runtime") == tree_fingerprint(source_home / "code/runtime") @@ -270,7 +299,7 @@ async def test_benchmark_entry_points_select_correct_defaults(benchmark, monkeyp @pytest.mark.parametrize("options", [{"sampling": True}, {"record_history": True}, {"max_evaluations": 1}]) -def test_test_runner_rejects_optimizer_modes(options): +def test_test_runner_rejects_optimizer_modes(home_eval, options): with pytest.raises(ValueError): home_eval.TestRunConfig.model_validate( {