fix(home): load runner credentials and localize dataset evaluation

This commit is contained in:
酱牛肉 2026-11-02 21:21:48 +08:00
parent e89930f9c2
commit 93898e314b
7 changed files with 513 additions and 235 deletions

View file

@ -20,6 +20,7 @@ from reme_home.judge import JUDGE_PLUGINS # noqa: E402
from reme_home.sampling import SamplingPolicy # noqa: E402
from reme.application import Application # noqa: E402
from reme.utils.env_utils import load_env # noqa: E402
# pylint: enable=wrong-import-position
@ -67,6 +68,7 @@ def load_config(config_path: str | Path, *, workspace_dir: str | None = None, pr
async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, instruction=None):
"""Apply the same runner configuration to each preparation or optimization lifecycle."""
workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir)
load_env(PROJECT_ROOT / ".env", override=False)
app = Application(
workspace_dir=str(workspace),
plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],

View file

@ -1,21 +1,226 @@
"""Evaluate a BEAM Home candidate in an independent test workspace."""
import argparse
import asyncio
import json
import sys
from pathlib import Path
from pydantic import Field, model_validator
import yaml
# Support direct execution from any working directory as well as module imports.
PROJECT_ROOT = Path(__file__).resolve().parents[2]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from benchmark.home_eval import main, run as run_evaluation # noqa: E402 # pylint: disable=wrong-import-position
# pylint: disable=wrong-import-position
from reme_home.code.repository import resolve_revision, validate_repository # noqa: E402
from reme_home.evaluation.artifacts import contained, read_json # noqa: E402
from reme_home.evaluation.component import blocking # noqa: E402
from reme_home.evaluation.inputs import EvaluationSpec, load_inputs # noqa: E402
from reme_home.judge import JUDGE_PLUGINS # noqa: E402
from reme_home.preparation.code import tree_fingerprint # noqa: E402
from reme_home.preparation.data.models import fingerprint # noqa: E402
from reme_home.preparation.evaluation_source import EvaluationSource # noqa: E402
from reme_home.preparation.spec import BenchmarkSpec, DebugSpec, SpecModel # noqa: E402
from reme.application import Application # noqa: E402
from reme.utils.env_utils import load_env # noqa: E402
# pylint: enable=wrong-import-position
DEFAULT_CONFIG = Path(__file__).with_name("config_home_eval.yaml")
class TestRunConfig(SpecModel):
"""Explicit dataset selection and execution options for one pinned candidate."""
source_workspace: str = Field(min_length=1)
test_workspace: str = Field(min_length=1)
code_commit: str = Field(min_length=4)
benchmark: BenchmarkSpec
exclude_training_cases: bool = False
evaluation: EvaluationSpec = Field(default_factory=lambda: EvaluationSpec(record_history=False))
@model_validator(mode="after")
def validate_test_mode(self):
"""Test runs score every prepared query and never consume optimizer budgets."""
options = self.evaluation
if any(
(
options.debug,
options.sampling,
options.record_history,
options.max_evaluations is not None,
options.sampling_policy is not None,
options.case_ids is not None,
options.query_ids is not None,
options.reuse_construction_from is not None,
),
):
raise ValueError(
"Test evaluation requires full scoring with record_history=false; select cases in benchmark",
)
if self.exclude_training_cases and (self.benchmark.name != "longmemeval" or not self.benchmark.all_cases):
raise ValueError("exclude_training_cases requires LongMemEval with all_cases=true")
return self
def project_path(value: str, project_root: Path) -> Path:
"""Resolve runner paths independently of the launching working directory."""
if value.startswith("~"):
raise ValueError("Use absolute or project-relative paths, not home-relative paths")
return (project_root / value).resolve()
def load_request(config_path, *, commit=None, source_workspace=None, test_workspace=None, project_root=PROJECT_ROOT):
"""Validate all source inputs before starting an application or writing test data."""
raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
for key, value in (
("code_commit", commit),
("source_workspace", source_workspace),
("test_workspace", test_workspace),
):
if value is not None:
raw[key] = value
# Execution settings can be partial while retaining the test-only history default.
raw["evaluation"] = {"record_history": False, **raw.get("evaluation", {})}
config = TestRunConfig.model_validate(raw)
source_path = project_path(config.source_workspace, project_root)
target = project_path(config.test_workspace, project_root)
if target.is_relative_to(source_path) or source_path.is_relative_to(target):
raise ValueError("Test and source workspaces must be separate, non-nested directories")
if target.exists() and (
not target.is_dir() or (any(target.iterdir()) and not (target / "home/metadata/workspace.json").is_file())
):
raise ValueError("Use an empty test workspace or a compatible prepared Home test workspace")
source = EvaluationSource.load(source_path, config.code_commit)
benchmark = config.benchmark.model_copy(deep=True)
if benchmark.name != source.spec.benchmark.name:
raise ValueError("Test benchmark must match the source experiment")
benchmark.source = str(project_path(benchmark.source, project_root).resolve(strict=True))
if benchmark.evidence_source is not None:
benchmark.evidence_source = str(project_path(benchmark.evidence_source, project_root).resolve(strict=True))
if config.exclude_training_cases:
items = json.loads(Path(benchmark.source).read_text(encoding="utf-8"))
ids = [str(item["question_id"]) for item in items]
if len(ids) != len(set(ids)):
raise ValueError("Duplicate LongMemEval question IDs")
training = set(source.record["dataset"]["case_ids"])
if training - set(ids):
raise ValueError("Test dataset is missing training IDs; provide the original full dataset")
benchmark.case_ids = [case_id for case_id in ids if case_id not in training]
benchmark.all_cases = False
benchmark = BenchmarkSpec.model_validate(benchmark.model_dump())
spec = source.spec.model_copy(deep=True)
spec.benchmark = benchmark
spec.debug = DebugSpec(enabled=False)
return config, target, source, spec
def existing_preparation(home, source, spec):
"""Reopen only the same request with unchanged code, runtime and normalized data."""
marker = contained(home, "metadata/preparation.json")
if not marker.exists():
return None
record = read_json(marker)
expected = {**source.provenance, "request_fingerprint": fingerprint(spec.model_dump(mode="json"))}
if record.get("test_evaluation") != expected:
raise ValueError("Test workspace belongs to a different candidate or dataset; use a new workspace")
domain = yaml.safe_load(contained(home, "domain_spec.yaml").read_text(encoding="utf-8"))
if fingerprint(domain) != record["config_fingerprint"]:
raise ValueError("Test preparation configuration has changed")
repository, _ = validate_repository(home, record)
if (
resolve_revision(repository, "HEAD") != record["code"]["initial_commit"]
or tree_fingerprint(repository) != record["code"]["candidate_fingerprint"]
or tree_fingerprint(contained(home, "code/runtime", directory=True)) != source.provenance["runtime_fingerprint"]
):
raise ValueError("Prepared test code or runtime has changed")
load_inputs(home, EvaluationSpec(record_history=False))
return record
async def run_evaluation(
config_path,
*,
commit=None,
source_workspace=None,
test_workspace=None,
prepare_only=False,
expected_benchmark=None,
):
"""Prepare transactionally, then score through the normal application lifecycle."""
config, target, source, spec = await blocking(
load_request,
config_path,
commit=commit,
source_workspace=source_workspace,
test_workspace=test_workspace,
)
if expected_benchmark is not None and config.benchmark.name != expected_benchmark:
raise ValueError(f"This entry point requires benchmark.name={expected_benchmark}")
load_env(PROJECT_ROOT / ".env", override=False)
app = Application(
workspace_dir=str(target),
plugins=["home", JUDGE_PLUGINS[config.benchmark.name]],
service={"backend": "cli"},
enable_logo=False,
log_to_file=False,
)
try:
await app.start()
home = target / "home"
prepared = await blocking(existing_preparation, home, source, spec)
if prepared is None:
preparation = app.context.components["home.preparation"]["default"]
prepared = await preparation.prepare(spec=spec.model_dump(mode="json"), code_source=source)
result = {
"test_workspace": str(target),
**source.provenance,
"evaluation_commit_sha": prepared["code"]["initial_commit"],
"case_count": prepared["dataset"]["case_count"],
"query_count": prepared["dataset"]["query_count"],
}
if not prepare_only:
response = await app.run_job(
"home_evaluate",
**config.evaluation.model_dump(mode="json", exclude_unset=True),
)
if not response.success:
raise RuntimeError(response.answer)
result["evaluation"] = response.answer
return result
finally:
await app.close()
async def run(config_path=DEFAULT_CONFIG, **kwargs):
"""Score the selected BEAM commit without changing its optimization workspace."""
"""Score the selected commit in its own Home workspace."""
return await run_evaluation(config_path, expected_benchmark="beam", **kwargs)
def main():
"""Explicit local one-shot entry point, with preparation-only service setup."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG)
parser.add_argument("--commit", help="Candidate commit SHA or unambiguous hexadecimal prefix")
parser.add_argument("--source-workspace", help="Prepared optimization workspace")
parser.add_argument("--test-workspace", help="Independent test output workspace")
parser.add_argument("--prepare-only", action="store_true", help="Prepare for a later home_evaluate service call")
args = parser.parse_args()
result = asyncio.run(
run(
args.config,
commit=args.commit,
source_workspace=args.source_workspace,
test_workspace=args.test_workspace,
prepare_only=args.prepare_only,
),
)
print(json.dumps(result, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main("beam", DEFAULT_CONFIG)
main()

View file

@ -1,207 +0,0 @@
"""Standalone test evaluation shared by the BEAM and LongMemEval entry points."""
import argparse
import asyncio
import json
from pathlib import Path
from pydantic import Field, model_validator
import yaml
from reme_home.code.repository import resolve_revision, validate_repository
from reme_home.evaluation.artifacts import contained, read_json
from reme_home.evaluation.component import blocking
from reme_home.evaluation.inputs import EvaluationSpec, load_inputs
from reme_home.judge import JUDGE_PLUGINS
from reme_home.preparation.code import tree_fingerprint
from reme_home.preparation.data.models import fingerprint
from reme_home.preparation.evaluation_source import EvaluationSource
from reme_home.preparation.spec import BenchmarkSpec, DebugSpec, SpecModel
from reme.application import Application
PROJECT_ROOT = Path(__file__).resolve().parents[1]
class TestRunConfig(SpecModel):
"""Explicit dataset selection and execution options for one pinned candidate."""
source_workspace: str = Field(min_length=1)
test_workspace: str = Field(min_length=1)
code_commit: str = Field(min_length=4)
benchmark: BenchmarkSpec
exclude_training_cases: bool = False
evaluation: EvaluationSpec = Field(default_factory=lambda: EvaluationSpec(record_history=False))
@model_validator(mode="after")
def validate_test_mode(self):
"""Test runs score every prepared query and never consume optimizer budgets."""
options = self.evaluation
if any(
(
options.debug,
options.sampling,
options.record_history,
options.max_evaluations is not None,
options.sampling_policy is not None,
options.case_ids is not None,
options.query_ids is not None,
options.reuse_construction_from is not None,
),
):
raise ValueError(
"Test evaluation requires full scoring with record_history=false; select cases in benchmark",
)
if self.exclude_training_cases and (self.benchmark.name != "longmemeval" or not self.benchmark.all_cases):
raise ValueError("exclude_training_cases requires LongMemEval with all_cases=true")
return self
def project_path(value: str, project_root: Path) -> Path:
"""Resolve runner paths independently of the launching working directory."""
if value.startswith("~"):
raise ValueError("Use absolute or project-relative paths, not home-relative paths")
return (project_root / value).resolve()
def load_request(config_path, *, commit=None, source_workspace=None, test_workspace=None, project_root=PROJECT_ROOT):
"""Validate all source inputs before starting an application or writing test data."""
raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
for key, value in (
("code_commit", commit),
("source_workspace", source_workspace),
("test_workspace", test_workspace),
):
if value is not None:
raw[key] = value
# Execution settings can be partial while retaining the test-only history default.
raw["evaluation"] = {"record_history": False, **raw.get("evaluation", {})}
config = TestRunConfig.model_validate(raw)
source_path = project_path(config.source_workspace, project_root)
target = project_path(config.test_workspace, project_root)
if target.is_relative_to(source_path) or source_path.is_relative_to(target):
raise ValueError("Test and source workspaces must be separate, non-nested directories")
if target.exists() and (
not target.is_dir() or (any(target.iterdir()) and not (target / "home/metadata/workspace.json").is_file())
):
raise ValueError("Use an empty test workspace or a compatible prepared Home test workspace")
source = EvaluationSource.load(source_path, config.code_commit)
benchmark = config.benchmark.model_copy(deep=True)
if benchmark.name != source.spec.benchmark.name:
raise ValueError("Test benchmark must match the source experiment")
benchmark.source = str(project_path(benchmark.source, project_root).resolve(strict=True))
if benchmark.evidence_source is not None:
benchmark.evidence_source = str(project_path(benchmark.evidence_source, project_root).resolve(strict=True))
if config.exclude_training_cases:
items = json.loads(Path(benchmark.source).read_text(encoding="utf-8"))
ids = [str(item["question_id"]) for item in items]
if len(ids) != len(set(ids)):
raise ValueError("Duplicate LongMemEval question IDs")
training = set(source.record["dataset"]["case_ids"])
if training - set(ids):
raise ValueError("Test dataset is missing training IDs; provide the original full dataset")
benchmark.case_ids = [case_id for case_id in ids if case_id not in training]
benchmark.all_cases = False
benchmark = BenchmarkSpec.model_validate(benchmark.model_dump())
spec = source.spec.model_copy(deep=True)
spec.benchmark = benchmark
spec.debug = DebugSpec(enabled=False)
return config, target, source, spec
def existing_preparation(home, source, spec):
"""Reopen only the same request with unchanged code, runtime and normalized data."""
marker = contained(home, "metadata/preparation.json")
if not marker.exists():
return None
record = read_json(marker)
expected = {**source.provenance, "request_fingerprint": fingerprint(spec.model_dump(mode="json"))}
if record.get("test_evaluation") != expected:
raise ValueError("Test workspace belongs to a different candidate or dataset; use a new workspace")
domain = yaml.safe_load(contained(home, "domain_spec.yaml").read_text(encoding="utf-8"))
if fingerprint(domain) != record["config_fingerprint"]:
raise ValueError("Test preparation configuration has changed")
repository, _ = validate_repository(home, record)
if (
resolve_revision(repository, "HEAD") != record["code"]["initial_commit"]
or tree_fingerprint(repository) != record["code"]["candidate_fingerprint"]
or tree_fingerprint(contained(home, "code/runtime", directory=True)) != source.provenance["runtime_fingerprint"]
):
raise ValueError("Prepared test code or runtime has changed")
load_inputs(home, EvaluationSpec(record_history=False))
return record
async def run(
config_path,
*,
commit=None,
source_workspace=None,
test_workspace=None,
prepare_only=False,
expected_benchmark=None,
):
"""Prepare transactionally, then score through the normal application lifecycle."""
config, target, source, spec = await blocking(
load_request,
config_path,
commit=commit,
source_workspace=source_workspace,
test_workspace=test_workspace,
)
if expected_benchmark is not None and config.benchmark.name != expected_benchmark:
raise ValueError(f"This entry point requires benchmark.name={expected_benchmark}")
app = Application(
workspace_dir=str(target),
plugins=["home", JUDGE_PLUGINS[config.benchmark.name]],
service={"backend": "cli"},
enable_logo=False,
log_to_file=False,
)
try:
await app.start()
home = target / "home"
prepared = await blocking(existing_preparation, home, source, spec)
if prepared is None:
preparation = app.context.components["home.preparation"]["default"]
prepared = await preparation.prepare(spec=spec.model_dump(mode="json"), code_source=source)
result = {
"test_workspace": str(target),
**source.provenance,
"evaluation_commit_sha": prepared["code"]["initial_commit"],
"case_count": prepared["dataset"]["case_count"],
"query_count": prepared["dataset"]["query_count"],
}
if not prepare_only:
response = await app.run_job(
"home_evaluate",
**config.evaluation.model_dump(mode="json", exclude_unset=True),
)
if not response.success:
raise RuntimeError(response.answer)
result["evaluation"] = response.answer
return result
finally:
await app.close()
def main(benchmark, default_config):
"""Explicit local one-shot entry point, with preparation-only service setup."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", type=Path, default=default_config)
parser.add_argument("--commit", help="Candidate commit SHA or unambiguous hexadecimal prefix")
parser.add_argument("--source-workspace", help="Prepared optimization workspace")
parser.add_argument("--test-workspace", help="Independent test output workspace")
parser.add_argument("--prepare-only", action="store_true", help="Prepare for a later home_evaluate service call")
args = parser.parse_args()
result = asyncio.run(
run(
args.config,
commit=args.commit,
source_workspace=args.source_workspace,
test_workspace=args.test_workspace,
prepare_only=args.prepare_only,
expected_benchmark=benchmark,
),
)
print(json.dumps(result, ensure_ascii=False, indent=2))

View file

@ -20,6 +20,7 @@ from reme_home.judge import JUDGE_PLUGINS # noqa: E402
from reme_home.sampling import SamplingPolicy # noqa: E402
from reme.application import Application # noqa: E402
from reme.utils.env_utils import load_env # noqa: E402
# pylint: enable=wrong-import-position
@ -67,6 +68,7 @@ def load_config(config_path: str | Path, *, workspace_dir: str | None = None, pr
async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, instruction=None):
"""Apply the same runner configuration to each preparation or optimization lifecycle."""
workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir)
load_env(PROJECT_ROOT / ".env", override=False)
app = Application(
workspace_dir=str(workspace),
plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],

View file

@ -1,21 +1,226 @@
"""Evaluate a LongMemEval Home candidate on an independent test dataset."""
"""Evaluate a LongMemEval Home candidate in an independent test workspace."""
import argparse
import asyncio
import json
import sys
from pathlib import Path
from pydantic import Field, model_validator
import yaml
# Support direct execution from any working directory as well as module imports.
PROJECT_ROOT = Path(__file__).resolve().parents[2]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from benchmark.home_eval import main, run as run_evaluation # noqa: E402 # pylint: disable=wrong-import-position
# pylint: disable=wrong-import-position
from reme_home.code.repository import resolve_revision, validate_repository # noqa: E402
from reme_home.evaluation.artifacts import contained, read_json # noqa: E402
from reme_home.evaluation.component import blocking # noqa: E402
from reme_home.evaluation.inputs import EvaluationSpec, load_inputs # noqa: E402
from reme_home.judge import JUDGE_PLUGINS # noqa: E402
from reme_home.preparation.code import tree_fingerprint # noqa: E402
from reme_home.preparation.data.models import fingerprint # noqa: E402
from reme_home.preparation.evaluation_source import EvaluationSource # noqa: E402
from reme_home.preparation.spec import BenchmarkSpec, DebugSpec, SpecModel # noqa: E402
from reme.application import Application # noqa: E402
from reme.utils.env_utils import load_env # noqa: E402
# pylint: enable=wrong-import-position
DEFAULT_CONFIG = Path(__file__).with_name("config_home_eval.yaml")
class TestRunConfig(SpecModel):
"""Explicit dataset selection and execution options for one pinned candidate."""
source_workspace: str = Field(min_length=1)
test_workspace: str = Field(min_length=1)
code_commit: str = Field(min_length=4)
benchmark: BenchmarkSpec
exclude_training_cases: bool = False
evaluation: EvaluationSpec = Field(default_factory=lambda: EvaluationSpec(record_history=False))
@model_validator(mode="after")
def validate_test_mode(self):
"""Test runs score every prepared query and never consume optimizer budgets."""
options = self.evaluation
if any(
(
options.debug,
options.sampling,
options.record_history,
options.max_evaluations is not None,
options.sampling_policy is not None,
options.case_ids is not None,
options.query_ids is not None,
options.reuse_construction_from is not None,
),
):
raise ValueError(
"Test evaluation requires full scoring with record_history=false; select cases in benchmark",
)
if self.exclude_training_cases and (self.benchmark.name != "longmemeval" or not self.benchmark.all_cases):
raise ValueError("exclude_training_cases requires LongMemEval with all_cases=true")
return self
def project_path(value: str, project_root: Path) -> Path:
"""Resolve runner paths independently of the launching working directory."""
if value.startswith("~"):
raise ValueError("Use absolute or project-relative paths, not home-relative paths")
return (project_root / value).resolve()
def load_request(config_path, *, commit=None, source_workspace=None, test_workspace=None, project_root=PROJECT_ROOT):
"""Validate all source inputs before starting an application or writing test data."""
raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
for key, value in (
("code_commit", commit),
("source_workspace", source_workspace),
("test_workspace", test_workspace),
):
if value is not None:
raw[key] = value
# Execution settings can be partial while retaining the test-only history default.
raw["evaluation"] = {"record_history": False, **raw.get("evaluation", {})}
config = TestRunConfig.model_validate(raw)
source_path = project_path(config.source_workspace, project_root)
target = project_path(config.test_workspace, project_root)
if target.is_relative_to(source_path) or source_path.is_relative_to(target):
raise ValueError("Test and source workspaces must be separate, non-nested directories")
if target.exists() and (
not target.is_dir() or (any(target.iterdir()) and not (target / "home/metadata/workspace.json").is_file())
):
raise ValueError("Use an empty test workspace or a compatible prepared Home test workspace")
source = EvaluationSource.load(source_path, config.code_commit)
benchmark = config.benchmark.model_copy(deep=True)
if benchmark.name != source.spec.benchmark.name:
raise ValueError("Test benchmark must match the source experiment")
benchmark.source = str(project_path(benchmark.source, project_root).resolve(strict=True))
if benchmark.evidence_source is not None:
benchmark.evidence_source = str(project_path(benchmark.evidence_source, project_root).resolve(strict=True))
if config.exclude_training_cases:
items = json.loads(Path(benchmark.source).read_text(encoding="utf-8"))
ids = [str(item["question_id"]) for item in items]
if len(ids) != len(set(ids)):
raise ValueError("Duplicate LongMemEval question IDs")
training = set(source.record["dataset"]["case_ids"])
if training - set(ids):
raise ValueError("Test dataset is missing training IDs; provide the original full dataset")
benchmark.case_ids = [case_id for case_id in ids if case_id not in training]
benchmark.all_cases = False
benchmark = BenchmarkSpec.model_validate(benchmark.model_dump())
spec = source.spec.model_copy(deep=True)
spec.benchmark = benchmark
spec.debug = DebugSpec(enabled=False)
return config, target, source, spec
def existing_preparation(home, source, spec):
"""Reopen only the same request with unchanged code, runtime and normalized data."""
marker = contained(home, "metadata/preparation.json")
if not marker.exists():
return None
record = read_json(marker)
expected = {**source.provenance, "request_fingerprint": fingerprint(spec.model_dump(mode="json"))}
if record.get("test_evaluation") != expected:
raise ValueError("Test workspace belongs to a different candidate or dataset; use a new workspace")
domain = yaml.safe_load(contained(home, "domain_spec.yaml").read_text(encoding="utf-8"))
if fingerprint(domain) != record["config_fingerprint"]:
raise ValueError("Test preparation configuration has changed")
repository, _ = validate_repository(home, record)
if (
resolve_revision(repository, "HEAD") != record["code"]["initial_commit"]
or tree_fingerprint(repository) != record["code"]["candidate_fingerprint"]
or tree_fingerprint(contained(home, "code/runtime", directory=True)) != source.provenance["runtime_fingerprint"]
):
raise ValueError("Prepared test code or runtime has changed")
load_inputs(home, EvaluationSpec(record_history=False))
return record
async def run_evaluation(
config_path,
*,
commit=None,
source_workspace=None,
test_workspace=None,
prepare_only=False,
expected_benchmark=None,
):
"""Prepare transactionally, then score through the normal application lifecycle."""
config, target, source, spec = await blocking(
load_request,
config_path,
commit=commit,
source_workspace=source_workspace,
test_workspace=test_workspace,
)
if expected_benchmark is not None and config.benchmark.name != expected_benchmark:
raise ValueError(f"This entry point requires benchmark.name={expected_benchmark}")
load_env(PROJECT_ROOT / ".env", override=False)
app = Application(
workspace_dir=str(target),
plugins=["home", JUDGE_PLUGINS[config.benchmark.name]],
service={"backend": "cli"},
enable_logo=False,
log_to_file=False,
)
try:
await app.start()
home = target / "home"
prepared = await blocking(existing_preparation, home, source, spec)
if prepared is None:
preparation = app.context.components["home.preparation"]["default"]
prepared = await preparation.prepare(spec=spec.model_dump(mode="json"), code_source=source)
result = {
"test_workspace": str(target),
**source.provenance,
"evaluation_commit_sha": prepared["code"]["initial_commit"],
"case_count": prepared["dataset"]["case_count"],
"query_count": prepared["dataset"]["query_count"],
}
if not prepare_only:
response = await app.run_job(
"home_evaluate",
**config.evaluation.model_dump(mode="json", exclude_unset=True),
)
if not response.success:
raise RuntimeError(response.answer)
result["evaluation"] = response.answer
return result
finally:
await app.close()
async def run(config_path=DEFAULT_CONFIG, **kwargs):
"""Score the selected LongMemEval commit in its own Home workspace."""
"""Score the selected commit in its own Home workspace."""
return await run_evaluation(config_path, expected_benchmark="longmemeval", **kwargs)
def main():
"""Explicit local one-shot entry point, with preparation-only service setup."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG)
parser.add_argument("--commit", help="Candidate commit SHA or unambiguous hexadecimal prefix")
parser.add_argument("--source-workspace", help="Prepared optimization workspace")
parser.add_argument("--test-workspace", help="Independent test output workspace")
parser.add_argument("--prepare-only", action="store_true", help="Prepare for a later home_evaluate service call")
args = parser.parse_args()
result = asyncio.run(
run(
args.config,
commit=args.commit,
source_workspace=args.source_workspace,
test_workspace=args.test_workspace,
prepare_only=args.prepare_only,
),
)
print(json.dumps(result, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main("longmemeval", DEFAULT_CONFIG)
main()

View file

@ -2,9 +2,11 @@
import hashlib
import json
import os
import sys
from importlib import import_module
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
@ -18,6 +20,46 @@ from benchmark.longmemeval import run_home as lme_runner
ROOT = Path(__file__).resolve().parents[3]
@pytest.fixture(autouse=True)
def isolated_runner_environment(tmp_path, monkeypatch):
"""Never load developer credentials while exercising benchmark lifecycles."""
for benchmark in ("beam", "longmemeval"):
for runner in ("run_home", "run_home_eval"):
module = import_module(f"benchmark.{benchmark}.{runner}")
monkeypatch.setattr(module, "PROJECT_ROOT", tmp_path)
@pytest.mark.asyncio
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
@pytest.mark.parametrize("runner", ["run_home", "run_home_eval"])
@pytest.mark.parametrize("env_file", [True, False])
async def test_credentials_loaded_before_application(tmp_path, monkeypatch, benchmark, runner, env_file):
"""Load repository .env without overriding exports, independently of cwd."""
module = import_module(f"benchmark.{benchmark}.{runner}")
monkeypatch.delenv("EMBEDDING_API_KEY", raising=False)
monkeypatch.setenv("LLM_API_KEY", "exported-key")
if env_file:
(tmp_path / ".env").write_text("LLM_API_KEY=file-key\nEMBEDDING_API_KEY=embedding-key\n")
cwd = tmp_path / "elsewhere"
cwd.mkdir()
monkeypatch.chdir(cwd)
if runner == "run_home_eval":
config = SimpleNamespace(benchmark=SimpleNamespace(name=benchmark))
monkeypatch.setattr(module, "load_request", lambda *args, **kwargs: (config, tmp_path, None, None))
class ApplicationReached(Exception):
"""Stop before creating any application resources."""
def factory(**_kwargs):
assert os.environ["LLM_API_KEY"] == "exported-key"
assert os.environ.get("EMBEDDING_API_KEY") == ("embedding-key" if env_file else None)
raise ApplicationReached
monkeypatch.setattr(module, "Application", factory)
with pytest.raises(ApplicationReached):
await module.run()
def test_preserved_lme_split_and_evidence():
"""Retain dev_meta's exact reviewed partition and annotations."""
directory = ROOT / "benchmark/longmemeval"

View file

@ -27,11 +27,17 @@ from test_preparation import ( # noqa: F401 # pylint: disable=unused-import
snapshot,
)
from benchmark import home_eval
from reme.schema import Response
from reme.components.agent_wrapper import AsAgentWrapper
@pytest.fixture(params=["beam", "longmemeval"])
def home_eval(request, tmp_path, monkeypatch):
module = import_module(f"benchmark.{request.param}.run_home_eval")
monkeypatch.setattr(module, "PROJECT_ROOT", tmp_path)
return module
def write_config(tmp_path, app, spec, commit):
config = {
"source_workspace": str(app.config.workspace_dir),
@ -54,7 +60,12 @@ async def prepare_source(app, inputs):
@pytest.mark.asyncio
async def test_import_historical_commit_ignores_dirty_checkout_and_missing_original_repo(app, inputs, tmp_path):
async def test_import_historical_commit_ignores_dirty_checkout_and_missing_original_repo(
home_eval,
app,
inputs,
tmp_path,
):
home, _ = await prepare_source(app, inputs)
repo = home / "code/repo"
command(repo, "switch", "-c", "candidate")
@ -72,7 +83,7 @@ async def test_import_historical_commit_ignores_dirty_checkout_and_missing_origi
path, _ = write_config(tmp_path, app, spec, selected[:10])
before = snapshot(home)
result = await home_eval.run(path, prepare_only=True)
result = await home_eval.run_evaluation(path, prepare_only=True)
assert result["source_commit_sha"] == selected
target = Path(result["test_workspace"]) / "home"
@ -88,10 +99,16 @@ async def test_import_historical_commit_ignores_dirty_checkout_and_missing_origi
@pytest.mark.asyncio
async def test_reopen_scores_all_queries_without_optimizer_history_and_closes(app, inputs, tmp_path, monkeypatch):
async def test_reopen_scores_all_queries_without_optimizer_history_and_closes(
home_eval,
app,
inputs,
tmp_path,
monkeypatch,
):
home, commit = await prepare_source(app, inputs)
path, _ = write_config(tmp_path, app, inputs[0], commit)
await home_eval.run(path, prepare_only=True)
await home_eval.run_evaluation(path, prepare_only=True)
applications = []
factory = home_eval.Application
@ -114,7 +131,7 @@ async def test_reopen_scores_all_queries_without_optimizer_history_and_closes(ap
return application
monkeypatch.setattr(home_eval, "Application", tracked_application)
result = await home_eval.run(path)
result = await home_eval.run_evaluation(path)
assert result["evaluation"] == {"result_dir": "test-results"}
applications[-1].close.assert_awaited_once()
assert not list((home / "evaluations").rglob("manifest.json"))
@ -123,7 +140,13 @@ async def test_reopen_scores_all_queries_without_optimizer_history_and_closes(ap
@pytest.mark.asyncio
@pytest.mark.parametrize("app", ["beam-judge", "lme-judge"], indirect=True)
async def test_full_evaluation_records_source_provenance_with_fake_docker(app, inputs, tmp_path, monkeypatch):
async def test_full_evaluation_records_source_provenance_with_fake_docker(
home_eval,
app,
inputs,
tmp_path,
monkeypatch,
):
spec, lme = inputs
if "lme-judge" in app.config.plugins:
spec["benchmark"] = {"name": "longmemeval", "source": str(lme), "all_cases": True}
@ -138,7 +161,7 @@ async def test_full_evaluation_records_source_provenance_with_fake_docker(app, i
answer = "yes" if spec["benchmark"]["name"] == "longmemeval" else '{"score": 0.5}'
monkeypatch.setattr(AsAgentWrapper, "reply", AsyncMock(return_value={"result": answer}))
before = snapshot(home)
result = await home_eval.run(path)
result = await home_eval.run_evaluation(path)
run = Path(result["evaluation"]["result_dir"])
manifest = read_json(run / "manifest.json")
assert manifest["test_evaluation"]["source_commit_sha"] == commit
@ -151,7 +174,13 @@ async def test_full_evaluation_records_source_provenance_with_fake_docker(app, i
@pytest.mark.asyncio
async def test_evaluation_failure_closes_application_and_preserves_preparation(app, inputs, tmp_path, monkeypatch):
async def test_evaluation_failure_closes_application_and_preserves_preparation(
home_eval,
app,
inputs,
tmp_path,
monkeypatch,
):
_, commit = await prepare_source(app, inputs)
path, config = write_config(tmp_path, app, inputs[0], commit)
factory = home_eval.Application
@ -166,14 +195,14 @@ async def test_evaluation_failure_closes_application_and_preserves_preparation(a
monkeypatch.setattr(home_eval, "Application", failing_application)
with pytest.raises(RuntimeError, match="Infrastructure failure"):
await home_eval.run(path)
await home_eval.run_evaluation(path)
applications[-1].close.assert_awaited_once()
assert (Path(config["test_workspace"]) / "home/metadata/preparation.json").is_file()
@pytest.mark.asyncio
@pytest.mark.parametrize("app", ["lme-judge"], indirect=True)
async def test_lme_default_selection_excludes_actual_training_cases(app, inputs, tmp_path):
async def test_lme_default_selection_excludes_actual_training_cases(home_eval, app, inputs, tmp_path):
spec, lme = inputs
original = json.loads(lme.read_text())
held_out = {**original[0], "question_id": "q2"}
@ -184,7 +213,7 @@ async def test_lme_default_selection_excludes_actual_training_cases(app, inputs,
config["benchmark"] = {"name": "longmemeval", "source": str(lme), "all_cases": True}
config["exclude_training_cases"] = True
path.write_text(yaml.safe_dump(config))
result = await home_eval.run(path, prepare_only=True, expected_benchmark="longmemeval")
result = await home_eval.run_evaluation(path, prepare_only=True, expected_benchmark="longmemeval")
target = Path(result["test_workspace"]) / "home"
_, dataset, cases = load_inputs(target, EvaluationSpec(record_history=False))
assert dataset.case_ids == ["q2"]
@ -194,10 +223,10 @@ async def test_lme_default_selection_excludes_actual_training_cases(app, inputs,
@pytest.mark.asyncio
@pytest.mark.parametrize("change", ["candidate", "dataset", "runtime", "test_code", "test_data"])
async def test_reopen_rejects_changed_identity_or_artifacts(app, inputs, tmp_path, change):
async def test_reopen_rejects_changed_identity_or_artifacts(home_eval, app, inputs, tmp_path, change):
_, commit = await prepare_source(app, inputs)
path, config = write_config(tmp_path, app, inputs[0], commit)
result = await home_eval.run(path, prepare_only=True)
result = await home_eval.run_evaluation(path, prepare_only=True)
home = Path(result["test_workspace"]) / "home"
if change == "candidate":
config["code_commit"] = "1234567890"
@ -216,13 +245,13 @@ async def test_reopen_rejects_changed_identity_or_artifacts(app, inputs, tmp_pat
file.write_text(json.dumps(case))
before = snapshot(home)
with pytest.raises((ValueError, CalledProcessError)):
await home_eval.run(path, prepare_only=True)
await home_eval.run_evaluation(path, prepare_only=True)
assert snapshot(home) == before
@pytest.mark.asyncio
@pytest.mark.parametrize("location", ["same", "child", "parent"])
async def test_rejects_overlapping_workspaces_before_start(app, inputs, tmp_path, monkeypatch, location):
async def test_rejects_overlapping_workspaces_before_start(home_eval, app, inputs, tmp_path, monkeypatch, location):
_, commit = await prepare_source(app, inputs)
path, config = write_config(tmp_path, app, inputs[0], commit)
source = Path(app.config.workspace_dir)
@ -232,12 +261,12 @@ async def test_rejects_overlapping_workspaces_before_start(app, inputs, tmp_path
factory = AsyncMock()
monkeypatch.setattr(home_eval, "Application", factory)
with pytest.raises(ValueError, match="non-nested"):
await home_eval.run(path)
await home_eval.run_evaluation(path)
factory.assert_not_called()
@pytest.mark.asyncio
async def test_import_runtime_drift_rolls_back_and_allows_retry(app, inputs, tmp_path, monkeypatch):
async def test_import_runtime_drift_rolls_back_and_allows_retry(home_eval, app, inputs, tmp_path, monkeypatch):
source_home, commit = await prepare_source(app, inputs)
path, config = write_config(tmp_path, app, inputs[0], commit)
install = EvaluationSource.install
@ -249,13 +278,13 @@ async def test_import_runtime_drift_rolls_back_and_allows_retry(app, inputs, tmp
monkeypatch.setattr(EvaluationSource, "install", fail_install)
with pytest.raises(ValueError, match="during import"):
await home_eval.run(path, prepare_only=True)
await home_eval.run_evaluation(path, prepare_only=True)
target = Path(config["test_workspace"]) / "home"
assert not (target / "metadata/preparation.json").exists()
assert not list((target / "code/runtime").iterdir())
assert not list((target / "code/repo").iterdir())
monkeypatch.setattr(EvaluationSource, "install", install)
await home_eval.run(path, prepare_only=True)
await home_eval.run_evaluation(path, prepare_only=True)
assert tree_fingerprint(target / "code/runtime") == tree_fingerprint(source_home / "code/runtime")
@ -270,7 +299,7 @@ async def test_benchmark_entry_points_select_correct_defaults(benchmark, monkeyp
@pytest.mark.parametrize("options", [{"sampling": True}, {"record_history": True}, {"max_evaluations": 1}])
def test_test_runner_rejects_optimizer_modes(options):
def test_test_runner_rejects_optimizer_modes(home_eval, options):
with pytest.raises(ValueError):
home_eval.TestRunConfig.model_validate(
{