mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-15 09:11:24 +00:00
fix(home): load runner credentials and localize dataset evaluation
This commit is contained in:
parent
e89930f9c2
commit
93898e314b
7 changed files with 513 additions and 235 deletions
|
|
@ -20,6 +20,7 @@ from reme_home.judge import JUDGE_PLUGINS # noqa: E402
|
|||
from reme_home.sampling import SamplingPolicy # noqa: E402
|
||||
|
||||
from reme.application import Application # noqa: E402
|
||||
from reme.utils.env_utils import load_env # noqa: E402
|
||||
|
||||
# pylint: enable=wrong-import-position
|
||||
|
||||
|
|
@ -67,6 +68,7 @@ def load_config(config_path: str | Path, *, workspace_dir: str | None = None, pr
|
|||
async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, instruction=None):
|
||||
"""Apply the same runner configuration to each preparation or optimization lifecycle."""
|
||||
workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir)
|
||||
load_env(PROJECT_ROOT / ".env", override=False)
|
||||
app = Application(
|
||||
workspace_dir=str(workspace),
|
||||
plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
|
||||
|
|
|
|||
|
|
@ -1,21 +1,226 @@
|
|||
"""Evaluate a BEAM Home candidate in an independent test workspace."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
import yaml
|
||||
|
||||
# Support direct execution from any working directory as well as module imports.
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from benchmark.home_eval import main, run as run_evaluation # noqa: E402 # pylint: disable=wrong-import-position
|
||||
# pylint: disable=wrong-import-position
|
||||
from reme_home.code.repository import resolve_revision, validate_repository # noqa: E402
|
||||
from reme_home.evaluation.artifacts import contained, read_json # noqa: E402
|
||||
from reme_home.evaluation.component import blocking # noqa: E402
|
||||
from reme_home.evaluation.inputs import EvaluationSpec, load_inputs # noqa: E402
|
||||
from reme_home.judge import JUDGE_PLUGINS # noqa: E402
|
||||
from reme_home.preparation.code import tree_fingerprint # noqa: E402
|
||||
from reme_home.preparation.data.models import fingerprint # noqa: E402
|
||||
from reme_home.preparation.evaluation_source import EvaluationSource # noqa: E402
|
||||
from reme_home.preparation.spec import BenchmarkSpec, DebugSpec, SpecModel # noqa: E402
|
||||
|
||||
from reme.application import Application # noqa: E402
|
||||
from reme.utils.env_utils import load_env # noqa: E402
|
||||
|
||||
# pylint: enable=wrong-import-position
|
||||
|
||||
DEFAULT_CONFIG = Path(__file__).with_name("config_home_eval.yaml")
|
||||
|
||||
|
||||
class TestRunConfig(SpecModel):
|
||||
"""Explicit dataset selection and execution options for one pinned candidate."""
|
||||
|
||||
source_workspace: str = Field(min_length=1)
|
||||
test_workspace: str = Field(min_length=1)
|
||||
code_commit: str = Field(min_length=4)
|
||||
benchmark: BenchmarkSpec
|
||||
exclude_training_cases: bool = False
|
||||
evaluation: EvaluationSpec = Field(default_factory=lambda: EvaluationSpec(record_history=False))
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_test_mode(self):
|
||||
"""Test runs score every prepared query and never consume optimizer budgets."""
|
||||
options = self.evaluation
|
||||
if any(
|
||||
(
|
||||
options.debug,
|
||||
options.sampling,
|
||||
options.record_history,
|
||||
options.max_evaluations is not None,
|
||||
options.sampling_policy is not None,
|
||||
options.case_ids is not None,
|
||||
options.query_ids is not None,
|
||||
options.reuse_construction_from is not None,
|
||||
),
|
||||
):
|
||||
raise ValueError(
|
||||
"Test evaluation requires full scoring with record_history=false; select cases in benchmark",
|
||||
)
|
||||
if self.exclude_training_cases and (self.benchmark.name != "longmemeval" or not self.benchmark.all_cases):
|
||||
raise ValueError("exclude_training_cases requires LongMemEval with all_cases=true")
|
||||
return self
|
||||
|
||||
|
||||
def project_path(value: str, project_root: Path) -> Path:
|
||||
"""Resolve runner paths independently of the launching working directory."""
|
||||
if value.startswith("~"):
|
||||
raise ValueError("Use absolute or project-relative paths, not home-relative paths")
|
||||
return (project_root / value).resolve()
|
||||
|
||||
|
||||
def load_request(config_path, *, commit=None, source_workspace=None, test_workspace=None, project_root=PROJECT_ROOT):
|
||||
"""Validate all source inputs before starting an application or writing test data."""
|
||||
raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
|
||||
for key, value in (
|
||||
("code_commit", commit),
|
||||
("source_workspace", source_workspace),
|
||||
("test_workspace", test_workspace),
|
||||
):
|
||||
if value is not None:
|
||||
raw[key] = value
|
||||
# Execution settings can be partial while retaining the test-only history default.
|
||||
raw["evaluation"] = {"record_history": False, **raw.get("evaluation", {})}
|
||||
config = TestRunConfig.model_validate(raw)
|
||||
source_path = project_path(config.source_workspace, project_root)
|
||||
target = project_path(config.test_workspace, project_root)
|
||||
if target.is_relative_to(source_path) or source_path.is_relative_to(target):
|
||||
raise ValueError("Test and source workspaces must be separate, non-nested directories")
|
||||
if target.exists() and (
|
||||
not target.is_dir() or (any(target.iterdir()) and not (target / "home/metadata/workspace.json").is_file())
|
||||
):
|
||||
raise ValueError("Use an empty test workspace or a compatible prepared Home test workspace")
|
||||
source = EvaluationSource.load(source_path, config.code_commit)
|
||||
benchmark = config.benchmark.model_copy(deep=True)
|
||||
if benchmark.name != source.spec.benchmark.name:
|
||||
raise ValueError("Test benchmark must match the source experiment")
|
||||
benchmark.source = str(project_path(benchmark.source, project_root).resolve(strict=True))
|
||||
if benchmark.evidence_source is not None:
|
||||
benchmark.evidence_source = str(project_path(benchmark.evidence_source, project_root).resolve(strict=True))
|
||||
if config.exclude_training_cases:
|
||||
items = json.loads(Path(benchmark.source).read_text(encoding="utf-8"))
|
||||
ids = [str(item["question_id"]) for item in items]
|
||||
if len(ids) != len(set(ids)):
|
||||
raise ValueError("Duplicate LongMemEval question IDs")
|
||||
training = set(source.record["dataset"]["case_ids"])
|
||||
if training - set(ids):
|
||||
raise ValueError("Test dataset is missing training IDs; provide the original full dataset")
|
||||
benchmark.case_ids = [case_id for case_id in ids if case_id not in training]
|
||||
benchmark.all_cases = False
|
||||
benchmark = BenchmarkSpec.model_validate(benchmark.model_dump())
|
||||
spec = source.spec.model_copy(deep=True)
|
||||
spec.benchmark = benchmark
|
||||
spec.debug = DebugSpec(enabled=False)
|
||||
return config, target, source, spec
|
||||
|
||||
|
||||
def existing_preparation(home, source, spec):
|
||||
"""Reopen only the same request with unchanged code, runtime and normalized data."""
|
||||
marker = contained(home, "metadata/preparation.json")
|
||||
if not marker.exists():
|
||||
return None
|
||||
record = read_json(marker)
|
||||
expected = {**source.provenance, "request_fingerprint": fingerprint(spec.model_dump(mode="json"))}
|
||||
if record.get("test_evaluation") != expected:
|
||||
raise ValueError("Test workspace belongs to a different candidate or dataset; use a new workspace")
|
||||
domain = yaml.safe_load(contained(home, "domain_spec.yaml").read_text(encoding="utf-8"))
|
||||
if fingerprint(domain) != record["config_fingerprint"]:
|
||||
raise ValueError("Test preparation configuration has changed")
|
||||
repository, _ = validate_repository(home, record)
|
||||
if (
|
||||
resolve_revision(repository, "HEAD") != record["code"]["initial_commit"]
|
||||
or tree_fingerprint(repository) != record["code"]["candidate_fingerprint"]
|
||||
or tree_fingerprint(contained(home, "code/runtime", directory=True)) != source.provenance["runtime_fingerprint"]
|
||||
):
|
||||
raise ValueError("Prepared test code or runtime has changed")
|
||||
load_inputs(home, EvaluationSpec(record_history=False))
|
||||
return record
|
||||
|
||||
|
||||
async def run_evaluation(
|
||||
config_path,
|
||||
*,
|
||||
commit=None,
|
||||
source_workspace=None,
|
||||
test_workspace=None,
|
||||
prepare_only=False,
|
||||
expected_benchmark=None,
|
||||
):
|
||||
"""Prepare transactionally, then score through the normal application lifecycle."""
|
||||
config, target, source, spec = await blocking(
|
||||
load_request,
|
||||
config_path,
|
||||
commit=commit,
|
||||
source_workspace=source_workspace,
|
||||
test_workspace=test_workspace,
|
||||
)
|
||||
if expected_benchmark is not None and config.benchmark.name != expected_benchmark:
|
||||
raise ValueError(f"This entry point requires benchmark.name={expected_benchmark}")
|
||||
load_env(PROJECT_ROOT / ".env", override=False)
|
||||
app = Application(
|
||||
workspace_dir=str(target),
|
||||
plugins=["home", JUDGE_PLUGINS[config.benchmark.name]],
|
||||
service={"backend": "cli"},
|
||||
enable_logo=False,
|
||||
log_to_file=False,
|
||||
)
|
||||
try:
|
||||
await app.start()
|
||||
home = target / "home"
|
||||
prepared = await blocking(existing_preparation, home, source, spec)
|
||||
if prepared is None:
|
||||
preparation = app.context.components["home.preparation"]["default"]
|
||||
prepared = await preparation.prepare(spec=spec.model_dump(mode="json"), code_source=source)
|
||||
result = {
|
||||
"test_workspace": str(target),
|
||||
**source.provenance,
|
||||
"evaluation_commit_sha": prepared["code"]["initial_commit"],
|
||||
"case_count": prepared["dataset"]["case_count"],
|
||||
"query_count": prepared["dataset"]["query_count"],
|
||||
}
|
||||
if not prepare_only:
|
||||
response = await app.run_job(
|
||||
"home_evaluate",
|
||||
**config.evaluation.model_dump(mode="json", exclude_unset=True),
|
||||
)
|
||||
if not response.success:
|
||||
raise RuntimeError(response.answer)
|
||||
result["evaluation"] = response.answer
|
||||
return result
|
||||
finally:
|
||||
await app.close()
|
||||
|
||||
|
||||
async def run(config_path=DEFAULT_CONFIG, **kwargs):
|
||||
"""Score the selected BEAM commit without changing its optimization workspace."""
|
||||
"""Score the selected commit in its own Home workspace."""
|
||||
return await run_evaluation(config_path, expected_benchmark="beam", **kwargs)
|
||||
|
||||
|
||||
def main():
|
||||
"""Explicit local one-shot entry point, with preparation-only service setup."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG)
|
||||
parser.add_argument("--commit", help="Candidate commit SHA or unambiguous hexadecimal prefix")
|
||||
parser.add_argument("--source-workspace", help="Prepared optimization workspace")
|
||||
parser.add_argument("--test-workspace", help="Independent test output workspace")
|
||||
parser.add_argument("--prepare-only", action="store_true", help="Prepare for a later home_evaluate service call")
|
||||
args = parser.parse_args()
|
||||
result = asyncio.run(
|
||||
run(
|
||||
args.config,
|
||||
commit=args.commit,
|
||||
source_workspace=args.source_workspace,
|
||||
test_workspace=args.test_workspace,
|
||||
prepare_only=args.prepare_only,
|
||||
),
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main("beam", DEFAULT_CONFIG)
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,207 +0,0 @@
|
|||
"""Standalone test evaluation shared by the BEAM and LongMemEval entry points."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
import yaml
|
||||
|
||||
from reme_home.code.repository import resolve_revision, validate_repository
|
||||
from reme_home.evaluation.artifacts import contained, read_json
|
||||
from reme_home.evaluation.component import blocking
|
||||
from reme_home.evaluation.inputs import EvaluationSpec, load_inputs
|
||||
from reme_home.judge import JUDGE_PLUGINS
|
||||
from reme_home.preparation.code import tree_fingerprint
|
||||
from reme_home.preparation.data.models import fingerprint
|
||||
from reme_home.preparation.evaluation_source import EvaluationSource
|
||||
from reme_home.preparation.spec import BenchmarkSpec, DebugSpec, SpecModel
|
||||
|
||||
from reme.application import Application
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
class TestRunConfig(SpecModel):
|
||||
"""Explicit dataset selection and execution options for one pinned candidate."""
|
||||
|
||||
source_workspace: str = Field(min_length=1)
|
||||
test_workspace: str = Field(min_length=1)
|
||||
code_commit: str = Field(min_length=4)
|
||||
benchmark: BenchmarkSpec
|
||||
exclude_training_cases: bool = False
|
||||
evaluation: EvaluationSpec = Field(default_factory=lambda: EvaluationSpec(record_history=False))
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_test_mode(self):
|
||||
"""Test runs score every prepared query and never consume optimizer budgets."""
|
||||
options = self.evaluation
|
||||
if any(
|
||||
(
|
||||
options.debug,
|
||||
options.sampling,
|
||||
options.record_history,
|
||||
options.max_evaluations is not None,
|
||||
options.sampling_policy is not None,
|
||||
options.case_ids is not None,
|
||||
options.query_ids is not None,
|
||||
options.reuse_construction_from is not None,
|
||||
),
|
||||
):
|
||||
raise ValueError(
|
||||
"Test evaluation requires full scoring with record_history=false; select cases in benchmark",
|
||||
)
|
||||
if self.exclude_training_cases and (self.benchmark.name != "longmemeval" or not self.benchmark.all_cases):
|
||||
raise ValueError("exclude_training_cases requires LongMemEval with all_cases=true")
|
||||
return self
|
||||
|
||||
|
||||
def project_path(value: str, project_root: Path) -> Path:
|
||||
"""Resolve runner paths independently of the launching working directory."""
|
||||
if value.startswith("~"):
|
||||
raise ValueError("Use absolute or project-relative paths, not home-relative paths")
|
||||
return (project_root / value).resolve()
|
||||
|
||||
|
||||
def load_request(config_path, *, commit=None, source_workspace=None, test_workspace=None, project_root=PROJECT_ROOT):
|
||||
"""Validate all source inputs before starting an application or writing test data."""
|
||||
raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
|
||||
for key, value in (
|
||||
("code_commit", commit),
|
||||
("source_workspace", source_workspace),
|
||||
("test_workspace", test_workspace),
|
||||
):
|
||||
if value is not None:
|
||||
raw[key] = value
|
||||
# Execution settings can be partial while retaining the test-only history default.
|
||||
raw["evaluation"] = {"record_history": False, **raw.get("evaluation", {})}
|
||||
config = TestRunConfig.model_validate(raw)
|
||||
source_path = project_path(config.source_workspace, project_root)
|
||||
target = project_path(config.test_workspace, project_root)
|
||||
if target.is_relative_to(source_path) or source_path.is_relative_to(target):
|
||||
raise ValueError("Test and source workspaces must be separate, non-nested directories")
|
||||
if target.exists() and (
|
||||
not target.is_dir() or (any(target.iterdir()) and not (target / "home/metadata/workspace.json").is_file())
|
||||
):
|
||||
raise ValueError("Use an empty test workspace or a compatible prepared Home test workspace")
|
||||
source = EvaluationSource.load(source_path, config.code_commit)
|
||||
benchmark = config.benchmark.model_copy(deep=True)
|
||||
if benchmark.name != source.spec.benchmark.name:
|
||||
raise ValueError("Test benchmark must match the source experiment")
|
||||
benchmark.source = str(project_path(benchmark.source, project_root).resolve(strict=True))
|
||||
if benchmark.evidence_source is not None:
|
||||
benchmark.evidence_source = str(project_path(benchmark.evidence_source, project_root).resolve(strict=True))
|
||||
if config.exclude_training_cases:
|
||||
items = json.loads(Path(benchmark.source).read_text(encoding="utf-8"))
|
||||
ids = [str(item["question_id"]) for item in items]
|
||||
if len(ids) != len(set(ids)):
|
||||
raise ValueError("Duplicate LongMemEval question IDs")
|
||||
training = set(source.record["dataset"]["case_ids"])
|
||||
if training - set(ids):
|
||||
raise ValueError("Test dataset is missing training IDs; provide the original full dataset")
|
||||
benchmark.case_ids = [case_id for case_id in ids if case_id not in training]
|
||||
benchmark.all_cases = False
|
||||
benchmark = BenchmarkSpec.model_validate(benchmark.model_dump())
|
||||
spec = source.spec.model_copy(deep=True)
|
||||
spec.benchmark = benchmark
|
||||
spec.debug = DebugSpec(enabled=False)
|
||||
return config, target, source, spec
|
||||
|
||||
|
||||
def existing_preparation(home, source, spec):
|
||||
"""Reopen only the same request with unchanged code, runtime and normalized data."""
|
||||
marker = contained(home, "metadata/preparation.json")
|
||||
if not marker.exists():
|
||||
return None
|
||||
record = read_json(marker)
|
||||
expected = {**source.provenance, "request_fingerprint": fingerprint(spec.model_dump(mode="json"))}
|
||||
if record.get("test_evaluation") != expected:
|
||||
raise ValueError("Test workspace belongs to a different candidate or dataset; use a new workspace")
|
||||
domain = yaml.safe_load(contained(home, "domain_spec.yaml").read_text(encoding="utf-8"))
|
||||
if fingerprint(domain) != record["config_fingerprint"]:
|
||||
raise ValueError("Test preparation configuration has changed")
|
||||
repository, _ = validate_repository(home, record)
|
||||
if (
|
||||
resolve_revision(repository, "HEAD") != record["code"]["initial_commit"]
|
||||
or tree_fingerprint(repository) != record["code"]["candidate_fingerprint"]
|
||||
or tree_fingerprint(contained(home, "code/runtime", directory=True)) != source.provenance["runtime_fingerprint"]
|
||||
):
|
||||
raise ValueError("Prepared test code or runtime has changed")
|
||||
load_inputs(home, EvaluationSpec(record_history=False))
|
||||
return record
|
||||
|
||||
|
||||
async def run(
|
||||
config_path,
|
||||
*,
|
||||
commit=None,
|
||||
source_workspace=None,
|
||||
test_workspace=None,
|
||||
prepare_only=False,
|
||||
expected_benchmark=None,
|
||||
):
|
||||
"""Prepare transactionally, then score through the normal application lifecycle."""
|
||||
config, target, source, spec = await blocking(
|
||||
load_request,
|
||||
config_path,
|
||||
commit=commit,
|
||||
source_workspace=source_workspace,
|
||||
test_workspace=test_workspace,
|
||||
)
|
||||
if expected_benchmark is not None and config.benchmark.name != expected_benchmark:
|
||||
raise ValueError(f"This entry point requires benchmark.name={expected_benchmark}")
|
||||
app = Application(
|
||||
workspace_dir=str(target),
|
||||
plugins=["home", JUDGE_PLUGINS[config.benchmark.name]],
|
||||
service={"backend": "cli"},
|
||||
enable_logo=False,
|
||||
log_to_file=False,
|
||||
)
|
||||
try:
|
||||
await app.start()
|
||||
home = target / "home"
|
||||
prepared = await blocking(existing_preparation, home, source, spec)
|
||||
if prepared is None:
|
||||
preparation = app.context.components["home.preparation"]["default"]
|
||||
prepared = await preparation.prepare(spec=spec.model_dump(mode="json"), code_source=source)
|
||||
result = {
|
||||
"test_workspace": str(target),
|
||||
**source.provenance,
|
||||
"evaluation_commit_sha": prepared["code"]["initial_commit"],
|
||||
"case_count": prepared["dataset"]["case_count"],
|
||||
"query_count": prepared["dataset"]["query_count"],
|
||||
}
|
||||
if not prepare_only:
|
||||
response = await app.run_job(
|
||||
"home_evaluate",
|
||||
**config.evaluation.model_dump(mode="json", exclude_unset=True),
|
||||
)
|
||||
if not response.success:
|
||||
raise RuntimeError(response.answer)
|
||||
result["evaluation"] = response.answer
|
||||
return result
|
||||
finally:
|
||||
await app.close()
|
||||
|
||||
|
||||
def main(benchmark, default_config):
|
||||
"""Explicit local one-shot entry point, with preparation-only service setup."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--config", type=Path, default=default_config)
|
||||
parser.add_argument("--commit", help="Candidate commit SHA or unambiguous hexadecimal prefix")
|
||||
parser.add_argument("--source-workspace", help="Prepared optimization workspace")
|
||||
parser.add_argument("--test-workspace", help="Independent test output workspace")
|
||||
parser.add_argument("--prepare-only", action="store_true", help="Prepare for a later home_evaluate service call")
|
||||
args = parser.parse_args()
|
||||
result = asyncio.run(
|
||||
run(
|
||||
args.config,
|
||||
commit=args.commit,
|
||||
source_workspace=args.source_workspace,
|
||||
test_workspace=args.test_workspace,
|
||||
prepare_only=args.prepare_only,
|
||||
expected_benchmark=benchmark,
|
||||
),
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
|
@ -20,6 +20,7 @@ from reme_home.judge import JUDGE_PLUGINS # noqa: E402
|
|||
from reme_home.sampling import SamplingPolicy # noqa: E402
|
||||
|
||||
from reme.application import Application # noqa: E402
|
||||
from reme.utils.env_utils import load_env # noqa: E402
|
||||
|
||||
# pylint: enable=wrong-import-position
|
||||
|
||||
|
|
@ -67,6 +68,7 @@ def load_config(config_path: str | Path, *, workspace_dir: str | None = None, pr
|
|||
async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, instruction=None):
|
||||
"""Apply the same runner configuration to each preparation or optimization lifecycle."""
|
||||
workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir)
|
||||
load_env(PROJECT_ROOT / ".env", override=False)
|
||||
app = Application(
|
||||
workspace_dir=str(workspace),
|
||||
plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
|
||||
|
|
|
|||
|
|
@ -1,21 +1,226 @@
|
|||
"""Evaluate a LongMemEval Home candidate on an independent test dataset."""
|
||||
"""Evaluate a LongMemEval Home candidate in an independent test workspace."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
import yaml
|
||||
|
||||
# Support direct execution from any working directory as well as module imports.
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from benchmark.home_eval import main, run as run_evaluation # noqa: E402 # pylint: disable=wrong-import-position
|
||||
# pylint: disable=wrong-import-position
|
||||
from reme_home.code.repository import resolve_revision, validate_repository # noqa: E402
|
||||
from reme_home.evaluation.artifacts import contained, read_json # noqa: E402
|
||||
from reme_home.evaluation.component import blocking # noqa: E402
|
||||
from reme_home.evaluation.inputs import EvaluationSpec, load_inputs # noqa: E402
|
||||
from reme_home.judge import JUDGE_PLUGINS # noqa: E402
|
||||
from reme_home.preparation.code import tree_fingerprint # noqa: E402
|
||||
from reme_home.preparation.data.models import fingerprint # noqa: E402
|
||||
from reme_home.preparation.evaluation_source import EvaluationSource # noqa: E402
|
||||
from reme_home.preparation.spec import BenchmarkSpec, DebugSpec, SpecModel # noqa: E402
|
||||
|
||||
from reme.application import Application # noqa: E402
|
||||
from reme.utils.env_utils import load_env # noqa: E402
|
||||
|
||||
# pylint: enable=wrong-import-position
|
||||
|
||||
DEFAULT_CONFIG = Path(__file__).with_name("config_home_eval.yaml")
|
||||
|
||||
|
||||
class TestRunConfig(SpecModel):
|
||||
"""Explicit dataset selection and execution options for one pinned candidate."""
|
||||
|
||||
source_workspace: str = Field(min_length=1)
|
||||
test_workspace: str = Field(min_length=1)
|
||||
code_commit: str = Field(min_length=4)
|
||||
benchmark: BenchmarkSpec
|
||||
exclude_training_cases: bool = False
|
||||
evaluation: EvaluationSpec = Field(default_factory=lambda: EvaluationSpec(record_history=False))
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_test_mode(self):
|
||||
"""Test runs score every prepared query and never consume optimizer budgets."""
|
||||
options = self.evaluation
|
||||
if any(
|
||||
(
|
||||
options.debug,
|
||||
options.sampling,
|
||||
options.record_history,
|
||||
options.max_evaluations is not None,
|
||||
options.sampling_policy is not None,
|
||||
options.case_ids is not None,
|
||||
options.query_ids is not None,
|
||||
options.reuse_construction_from is not None,
|
||||
),
|
||||
):
|
||||
raise ValueError(
|
||||
"Test evaluation requires full scoring with record_history=false; select cases in benchmark",
|
||||
)
|
||||
if self.exclude_training_cases and (self.benchmark.name != "longmemeval" or not self.benchmark.all_cases):
|
||||
raise ValueError("exclude_training_cases requires LongMemEval with all_cases=true")
|
||||
return self
|
||||
|
||||
|
||||
def project_path(value: str, project_root: Path) -> Path:
|
||||
"""Resolve runner paths independently of the launching working directory."""
|
||||
if value.startswith("~"):
|
||||
raise ValueError("Use absolute or project-relative paths, not home-relative paths")
|
||||
return (project_root / value).resolve()
|
||||
|
||||
|
||||
def load_request(config_path, *, commit=None, source_workspace=None, test_workspace=None, project_root=PROJECT_ROOT):
|
||||
"""Validate all source inputs before starting an application or writing test data."""
|
||||
raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
|
||||
for key, value in (
|
||||
("code_commit", commit),
|
||||
("source_workspace", source_workspace),
|
||||
("test_workspace", test_workspace),
|
||||
):
|
||||
if value is not None:
|
||||
raw[key] = value
|
||||
# Execution settings can be partial while retaining the test-only history default.
|
||||
raw["evaluation"] = {"record_history": False, **raw.get("evaluation", {})}
|
||||
config = TestRunConfig.model_validate(raw)
|
||||
source_path = project_path(config.source_workspace, project_root)
|
||||
target = project_path(config.test_workspace, project_root)
|
||||
if target.is_relative_to(source_path) or source_path.is_relative_to(target):
|
||||
raise ValueError("Test and source workspaces must be separate, non-nested directories")
|
||||
if target.exists() and (
|
||||
not target.is_dir() or (any(target.iterdir()) and not (target / "home/metadata/workspace.json").is_file())
|
||||
):
|
||||
raise ValueError("Use an empty test workspace or a compatible prepared Home test workspace")
|
||||
source = EvaluationSource.load(source_path, config.code_commit)
|
||||
benchmark = config.benchmark.model_copy(deep=True)
|
||||
if benchmark.name != source.spec.benchmark.name:
|
||||
raise ValueError("Test benchmark must match the source experiment")
|
||||
benchmark.source = str(project_path(benchmark.source, project_root).resolve(strict=True))
|
||||
if benchmark.evidence_source is not None:
|
||||
benchmark.evidence_source = str(project_path(benchmark.evidence_source, project_root).resolve(strict=True))
|
||||
if config.exclude_training_cases:
|
||||
items = json.loads(Path(benchmark.source).read_text(encoding="utf-8"))
|
||||
ids = [str(item["question_id"]) for item in items]
|
||||
if len(ids) != len(set(ids)):
|
||||
raise ValueError("Duplicate LongMemEval question IDs")
|
||||
training = set(source.record["dataset"]["case_ids"])
|
||||
if training - set(ids):
|
||||
raise ValueError("Test dataset is missing training IDs; provide the original full dataset")
|
||||
benchmark.case_ids = [case_id for case_id in ids if case_id not in training]
|
||||
benchmark.all_cases = False
|
||||
benchmark = BenchmarkSpec.model_validate(benchmark.model_dump())
|
||||
spec = source.spec.model_copy(deep=True)
|
||||
spec.benchmark = benchmark
|
||||
spec.debug = DebugSpec(enabled=False)
|
||||
return config, target, source, spec
|
||||
|
||||
|
||||
def existing_preparation(home, source, spec):
|
||||
"""Reopen only the same request with unchanged code, runtime and normalized data."""
|
||||
marker = contained(home, "metadata/preparation.json")
|
||||
if not marker.exists():
|
||||
return None
|
||||
record = read_json(marker)
|
||||
expected = {**source.provenance, "request_fingerprint": fingerprint(spec.model_dump(mode="json"))}
|
||||
if record.get("test_evaluation") != expected:
|
||||
raise ValueError("Test workspace belongs to a different candidate or dataset; use a new workspace")
|
||||
domain = yaml.safe_load(contained(home, "domain_spec.yaml").read_text(encoding="utf-8"))
|
||||
if fingerprint(domain) != record["config_fingerprint"]:
|
||||
raise ValueError("Test preparation configuration has changed")
|
||||
repository, _ = validate_repository(home, record)
|
||||
if (
|
||||
resolve_revision(repository, "HEAD") != record["code"]["initial_commit"]
|
||||
or tree_fingerprint(repository) != record["code"]["candidate_fingerprint"]
|
||||
or tree_fingerprint(contained(home, "code/runtime", directory=True)) != source.provenance["runtime_fingerprint"]
|
||||
):
|
||||
raise ValueError("Prepared test code or runtime has changed")
|
||||
load_inputs(home, EvaluationSpec(record_history=False))
|
||||
return record
|
||||
|
||||
|
||||
async def run_evaluation(
|
||||
config_path,
|
||||
*,
|
||||
commit=None,
|
||||
source_workspace=None,
|
||||
test_workspace=None,
|
||||
prepare_only=False,
|
||||
expected_benchmark=None,
|
||||
):
|
||||
"""Prepare transactionally, then score through the normal application lifecycle."""
|
||||
config, target, source, spec = await blocking(
|
||||
load_request,
|
||||
config_path,
|
||||
commit=commit,
|
||||
source_workspace=source_workspace,
|
||||
test_workspace=test_workspace,
|
||||
)
|
||||
if expected_benchmark is not None and config.benchmark.name != expected_benchmark:
|
||||
raise ValueError(f"This entry point requires benchmark.name={expected_benchmark}")
|
||||
load_env(PROJECT_ROOT / ".env", override=False)
|
||||
app = Application(
|
||||
workspace_dir=str(target),
|
||||
plugins=["home", JUDGE_PLUGINS[config.benchmark.name]],
|
||||
service={"backend": "cli"},
|
||||
enable_logo=False,
|
||||
log_to_file=False,
|
||||
)
|
||||
try:
|
||||
await app.start()
|
||||
home = target / "home"
|
||||
prepared = await blocking(existing_preparation, home, source, spec)
|
||||
if prepared is None:
|
||||
preparation = app.context.components["home.preparation"]["default"]
|
||||
prepared = await preparation.prepare(spec=spec.model_dump(mode="json"), code_source=source)
|
||||
result = {
|
||||
"test_workspace": str(target),
|
||||
**source.provenance,
|
||||
"evaluation_commit_sha": prepared["code"]["initial_commit"],
|
||||
"case_count": prepared["dataset"]["case_count"],
|
||||
"query_count": prepared["dataset"]["query_count"],
|
||||
}
|
||||
if not prepare_only:
|
||||
response = await app.run_job(
|
||||
"home_evaluate",
|
||||
**config.evaluation.model_dump(mode="json", exclude_unset=True),
|
||||
)
|
||||
if not response.success:
|
||||
raise RuntimeError(response.answer)
|
||||
result["evaluation"] = response.answer
|
||||
return result
|
||||
finally:
|
||||
await app.close()
|
||||
|
||||
|
||||
async def run(config_path=DEFAULT_CONFIG, **kwargs):
|
||||
"""Score the selected LongMemEval commit in its own Home workspace."""
|
||||
"""Score the selected commit in its own Home workspace."""
|
||||
return await run_evaluation(config_path, expected_benchmark="longmemeval", **kwargs)
|
||||
|
||||
|
||||
def main():
|
||||
"""Explicit local one-shot entry point, with preparation-only service setup."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG)
|
||||
parser.add_argument("--commit", help="Candidate commit SHA or unambiguous hexadecimal prefix")
|
||||
parser.add_argument("--source-workspace", help="Prepared optimization workspace")
|
||||
parser.add_argument("--test-workspace", help="Independent test output workspace")
|
||||
parser.add_argument("--prepare-only", action="store_true", help="Prepare for a later home_evaluate service call")
|
||||
args = parser.parse_args()
|
||||
result = asyncio.run(
|
||||
run(
|
||||
args.config,
|
||||
commit=args.commit,
|
||||
source_workspace=args.source_workspace,
|
||||
test_workspace=args.test_workspace,
|
||||
prepare_only=args.prepare_only,
|
||||
),
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main("longmemeval", DEFAULT_CONFIG)
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -2,9 +2,11 @@
|
|||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -18,6 +20,46 @@ from benchmark.longmemeval import run_home as lme_runner
|
|||
ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolated_runner_environment(tmp_path, monkeypatch):
|
||||
"""Never load developer credentials while exercising benchmark lifecycles."""
|
||||
for benchmark in ("beam", "longmemeval"):
|
||||
for runner in ("run_home", "run_home_eval"):
|
||||
module = import_module(f"benchmark.{benchmark}.{runner}")
|
||||
monkeypatch.setattr(module, "PROJECT_ROOT", tmp_path)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
|
||||
@pytest.mark.parametrize("runner", ["run_home", "run_home_eval"])
|
||||
@pytest.mark.parametrize("env_file", [True, False])
|
||||
async def test_credentials_loaded_before_application(tmp_path, monkeypatch, benchmark, runner, env_file):
|
||||
"""Load repository .env without overriding exports, independently of cwd."""
|
||||
module = import_module(f"benchmark.{benchmark}.{runner}")
|
||||
monkeypatch.delenv("EMBEDDING_API_KEY", raising=False)
|
||||
monkeypatch.setenv("LLM_API_KEY", "exported-key")
|
||||
if env_file:
|
||||
(tmp_path / ".env").write_text("LLM_API_KEY=file-key\nEMBEDDING_API_KEY=embedding-key\n")
|
||||
cwd = tmp_path / "elsewhere"
|
||||
cwd.mkdir()
|
||||
monkeypatch.chdir(cwd)
|
||||
if runner == "run_home_eval":
|
||||
config = SimpleNamespace(benchmark=SimpleNamespace(name=benchmark))
|
||||
monkeypatch.setattr(module, "load_request", lambda *args, **kwargs: (config, tmp_path, None, None))
|
||||
|
||||
class ApplicationReached(Exception):
|
||||
"""Stop before creating any application resources."""
|
||||
|
||||
def factory(**_kwargs):
|
||||
assert os.environ["LLM_API_KEY"] == "exported-key"
|
||||
assert os.environ.get("EMBEDDING_API_KEY") == ("embedding-key" if env_file else None)
|
||||
raise ApplicationReached
|
||||
|
||||
monkeypatch.setattr(module, "Application", factory)
|
||||
with pytest.raises(ApplicationReached):
|
||||
await module.run()
|
||||
|
||||
|
||||
def test_preserved_lme_split_and_evidence():
|
||||
"""Retain dev_meta's exact reviewed partition and annotations."""
|
||||
directory = ROOT / "benchmark/longmemeval"
|
||||
|
|
|
|||
|
|
@ -27,11 +27,17 @@ from test_preparation import ( # noqa: F401 # pylint: disable=unused-import
|
|||
snapshot,
|
||||
)
|
||||
|
||||
from benchmark import home_eval
|
||||
from reme.schema import Response
|
||||
from reme.components.agent_wrapper import AsAgentWrapper
|
||||
|
||||
|
||||
@pytest.fixture(params=["beam", "longmemeval"])
|
||||
def home_eval(request, tmp_path, monkeypatch):
|
||||
module = import_module(f"benchmark.{request.param}.run_home_eval")
|
||||
monkeypatch.setattr(module, "PROJECT_ROOT", tmp_path)
|
||||
return module
|
||||
|
||||
|
||||
def write_config(tmp_path, app, spec, commit):
|
||||
config = {
|
||||
"source_workspace": str(app.config.workspace_dir),
|
||||
|
|
@ -54,7 +60,12 @@ async def prepare_source(app, inputs):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_import_historical_commit_ignores_dirty_checkout_and_missing_original_repo(app, inputs, tmp_path):
|
||||
async def test_import_historical_commit_ignores_dirty_checkout_and_missing_original_repo(
|
||||
home_eval,
|
||||
app,
|
||||
inputs,
|
||||
tmp_path,
|
||||
):
|
||||
home, _ = await prepare_source(app, inputs)
|
||||
repo = home / "code/repo"
|
||||
command(repo, "switch", "-c", "candidate")
|
||||
|
|
@ -72,7 +83,7 @@ async def test_import_historical_commit_ignores_dirty_checkout_and_missing_origi
|
|||
path, _ = write_config(tmp_path, app, spec, selected[:10])
|
||||
before = snapshot(home)
|
||||
|
||||
result = await home_eval.run(path, prepare_only=True)
|
||||
result = await home_eval.run_evaluation(path, prepare_only=True)
|
||||
|
||||
assert result["source_commit_sha"] == selected
|
||||
target = Path(result["test_workspace"]) / "home"
|
||||
|
|
@ -88,10 +99,16 @@ async def test_import_historical_commit_ignores_dirty_checkout_and_missing_origi
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reopen_scores_all_queries_without_optimizer_history_and_closes(app, inputs, tmp_path, monkeypatch):
|
||||
async def test_reopen_scores_all_queries_without_optimizer_history_and_closes(
|
||||
home_eval,
|
||||
app,
|
||||
inputs,
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
home, commit = await prepare_source(app, inputs)
|
||||
path, _ = write_config(tmp_path, app, inputs[0], commit)
|
||||
await home_eval.run(path, prepare_only=True)
|
||||
await home_eval.run_evaluation(path, prepare_only=True)
|
||||
applications = []
|
||||
factory = home_eval.Application
|
||||
|
||||
|
|
@ -114,7 +131,7 @@ async def test_reopen_scores_all_queries_without_optimizer_history_and_closes(ap
|
|||
return application
|
||||
|
||||
monkeypatch.setattr(home_eval, "Application", tracked_application)
|
||||
result = await home_eval.run(path)
|
||||
result = await home_eval.run_evaluation(path)
|
||||
assert result["evaluation"] == {"result_dir": "test-results"}
|
||||
applications[-1].close.assert_awaited_once()
|
||||
assert not list((home / "evaluations").rglob("manifest.json"))
|
||||
|
|
@ -123,7 +140,13 @@ async def test_reopen_scores_all_queries_without_optimizer_history_and_closes(ap
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("app", ["beam-judge", "lme-judge"], indirect=True)
|
||||
async def test_full_evaluation_records_source_provenance_with_fake_docker(app, inputs, tmp_path, monkeypatch):
|
||||
async def test_full_evaluation_records_source_provenance_with_fake_docker(
|
||||
home_eval,
|
||||
app,
|
||||
inputs,
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
spec, lme = inputs
|
||||
if "lme-judge" in app.config.plugins:
|
||||
spec["benchmark"] = {"name": "longmemeval", "source": str(lme), "all_cases": True}
|
||||
|
|
@ -138,7 +161,7 @@ async def test_full_evaluation_records_source_provenance_with_fake_docker(app, i
|
|||
answer = "yes" if spec["benchmark"]["name"] == "longmemeval" else '{"score": 0.5}'
|
||||
monkeypatch.setattr(AsAgentWrapper, "reply", AsyncMock(return_value={"result": answer}))
|
||||
before = snapshot(home)
|
||||
result = await home_eval.run(path)
|
||||
result = await home_eval.run_evaluation(path)
|
||||
run = Path(result["evaluation"]["result_dir"])
|
||||
manifest = read_json(run / "manifest.json")
|
||||
assert manifest["test_evaluation"]["source_commit_sha"] == commit
|
||||
|
|
@ -151,7 +174,13 @@ async def test_full_evaluation_records_source_provenance_with_fake_docker(app, i
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_evaluation_failure_closes_application_and_preserves_preparation(app, inputs, tmp_path, monkeypatch):
|
||||
async def test_evaluation_failure_closes_application_and_preserves_preparation(
|
||||
home_eval,
|
||||
app,
|
||||
inputs,
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
_, commit = await prepare_source(app, inputs)
|
||||
path, config = write_config(tmp_path, app, inputs[0], commit)
|
||||
factory = home_eval.Application
|
||||
|
|
@ -166,14 +195,14 @@ async def test_evaluation_failure_closes_application_and_preserves_preparation(a
|
|||
|
||||
monkeypatch.setattr(home_eval, "Application", failing_application)
|
||||
with pytest.raises(RuntimeError, match="Infrastructure failure"):
|
||||
await home_eval.run(path)
|
||||
await home_eval.run_evaluation(path)
|
||||
applications[-1].close.assert_awaited_once()
|
||||
assert (Path(config["test_workspace"]) / "home/metadata/preparation.json").is_file()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("app", ["lme-judge"], indirect=True)
|
||||
async def test_lme_default_selection_excludes_actual_training_cases(app, inputs, tmp_path):
|
||||
async def test_lme_default_selection_excludes_actual_training_cases(home_eval, app, inputs, tmp_path):
|
||||
spec, lme = inputs
|
||||
original = json.loads(lme.read_text())
|
||||
held_out = {**original[0], "question_id": "q2"}
|
||||
|
|
@ -184,7 +213,7 @@ async def test_lme_default_selection_excludes_actual_training_cases(app, inputs,
|
|||
config["benchmark"] = {"name": "longmemeval", "source": str(lme), "all_cases": True}
|
||||
config["exclude_training_cases"] = True
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
result = await home_eval.run(path, prepare_only=True, expected_benchmark="longmemeval")
|
||||
result = await home_eval.run_evaluation(path, prepare_only=True, expected_benchmark="longmemeval")
|
||||
target = Path(result["test_workspace"]) / "home"
|
||||
_, dataset, cases = load_inputs(target, EvaluationSpec(record_history=False))
|
||||
assert dataset.case_ids == ["q2"]
|
||||
|
|
@ -194,10 +223,10 @@ async def test_lme_default_selection_excludes_actual_training_cases(app, inputs,
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("change", ["candidate", "dataset", "runtime", "test_code", "test_data"])
|
||||
async def test_reopen_rejects_changed_identity_or_artifacts(app, inputs, tmp_path, change):
|
||||
async def test_reopen_rejects_changed_identity_or_artifacts(home_eval, app, inputs, tmp_path, change):
|
||||
_, commit = await prepare_source(app, inputs)
|
||||
path, config = write_config(tmp_path, app, inputs[0], commit)
|
||||
result = await home_eval.run(path, prepare_only=True)
|
||||
result = await home_eval.run_evaluation(path, prepare_only=True)
|
||||
home = Path(result["test_workspace"]) / "home"
|
||||
if change == "candidate":
|
||||
config["code_commit"] = "1234567890"
|
||||
|
|
@ -216,13 +245,13 @@ async def test_reopen_rejects_changed_identity_or_artifacts(app, inputs, tmp_pat
|
|||
file.write_text(json.dumps(case))
|
||||
before = snapshot(home)
|
||||
with pytest.raises((ValueError, CalledProcessError)):
|
||||
await home_eval.run(path, prepare_only=True)
|
||||
await home_eval.run_evaluation(path, prepare_only=True)
|
||||
assert snapshot(home) == before
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("location", ["same", "child", "parent"])
|
||||
async def test_rejects_overlapping_workspaces_before_start(app, inputs, tmp_path, monkeypatch, location):
|
||||
async def test_rejects_overlapping_workspaces_before_start(home_eval, app, inputs, tmp_path, monkeypatch, location):
|
||||
_, commit = await prepare_source(app, inputs)
|
||||
path, config = write_config(tmp_path, app, inputs[0], commit)
|
||||
source = Path(app.config.workspace_dir)
|
||||
|
|
@ -232,12 +261,12 @@ async def test_rejects_overlapping_workspaces_before_start(app, inputs, tmp_path
|
|||
factory = AsyncMock()
|
||||
monkeypatch.setattr(home_eval, "Application", factory)
|
||||
with pytest.raises(ValueError, match="non-nested"):
|
||||
await home_eval.run(path)
|
||||
await home_eval.run_evaluation(path)
|
||||
factory.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_import_runtime_drift_rolls_back_and_allows_retry(app, inputs, tmp_path, monkeypatch):
|
||||
async def test_import_runtime_drift_rolls_back_and_allows_retry(home_eval, app, inputs, tmp_path, monkeypatch):
|
||||
source_home, commit = await prepare_source(app, inputs)
|
||||
path, config = write_config(tmp_path, app, inputs[0], commit)
|
||||
install = EvaluationSource.install
|
||||
|
|
@ -249,13 +278,13 @@ async def test_import_runtime_drift_rolls_back_and_allows_retry(app, inputs, tmp
|
|||
|
||||
monkeypatch.setattr(EvaluationSource, "install", fail_install)
|
||||
with pytest.raises(ValueError, match="during import"):
|
||||
await home_eval.run(path, prepare_only=True)
|
||||
await home_eval.run_evaluation(path, prepare_only=True)
|
||||
target = Path(config["test_workspace"]) / "home"
|
||||
assert not (target / "metadata/preparation.json").exists()
|
||||
assert not list((target / "code/runtime").iterdir())
|
||||
assert not list((target / "code/repo").iterdir())
|
||||
monkeypatch.setattr(EvaluationSource, "install", install)
|
||||
await home_eval.run(path, prepare_only=True)
|
||||
await home_eval.run_evaluation(path, prepare_only=True)
|
||||
assert tree_fingerprint(target / "code/runtime") == tree_fingerprint(source_home / "code/runtime")
|
||||
|
||||
|
||||
|
|
@ -270,7 +299,7 @@ async def test_benchmark_entry_points_select_correct_defaults(benchmark, monkeyp
|
|||
|
||||
|
||||
@pytest.mark.parametrize("options", [{"sampling": True}, {"record_history": True}, {"max_evaluations": 1}])
|
||||
def test_test_runner_rejects_optimizer_modes(options):
|
||||
def test_test_runner_rejects_optimizer_modes(home_eval, options):
|
||||
with pytest.raises(ValueError):
|
||||
home_eval.TestRunConfig.model_validate(
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue