diff --git a/benchmark/beam/README.md b/benchmark/beam/README.md index 0fc26f72..798eaa98 100644 --- a/benchmark/beam/README.md +++ b/benchmark/beam/README.md @@ -168,7 +168,7 @@ The top-level `validation_cycle_time_budget_minutes` configures the host optimiz advisory time between completed formal evaluations. It defaults to 150 minutes; set a positive number to change it or null to disable timed reminders. -The top-level `max_validations` sets the optimizer's formal-evaluation budget +The top-level `max_evaluations` sets the optimizer's formal-evaluation budget (positive integer, default 10). A slot is consumed at first sandbox startup, including later failures or cancellations; initial baseline and debug are excluded. diff --git a/benchmark/beam/README_ZH.md b/benchmark/beam/README_ZH.md index a10ed627..9647fc37 100644 --- a/benchmark/beam/README_ZH.md +++ b/benchmark/beam/README_ZH.md @@ -157,7 +157,7 @@ Python 中也可调用 `await benchmark.beam.run_home.run(workspace_dir=...)`。 顶层 `validation_cycle_time_budget_minutes` 配置宿主 optimizer 两次已完成正式评测之间的 建议用时,默认 150 分钟;可改为其他正数,或设为 null 关闭时间提醒。 -顶层 `max_validations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。 +顶层 `max_evaluations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。 首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。 准备完成后,用同一份配置进行本地一次性优化: diff --git a/benchmark/beam/config_home.yaml b/benchmark/beam/config_home.yaml index 6011264a..fa3fb901 100644 --- a/benchmark/beam/config_home.yaml +++ b/benchmark/beam/config_home.yaml @@ -2,7 +2,7 @@ # Paths are relative to the ReMe repository root, not the cwd. workspace_dir: benchmark/beam/workspaces/home-100K # Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug. -max_validations: 10 +max_evaluations: 10 # Target minutes between completed formal optimizer evaluations; null disables timed reminders. validation_cycle_time_budget_minutes: 150 # Policy defaults for Home evaluation; each call still controls sampling=true/false. diff --git a/benchmark/beam/run_home.py b/benchmark/beam/run_home.py index d951dfcc..b231f64d 100644 --- a/benchmark/beam/run_home.py +++ b/benchmark/beam/run_home.py @@ -34,7 +34,7 @@ class RunHomeConfig(BaseModel): preparation: dict sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy) validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False) - max_validations: int = Field(default=10, ge=1) + max_evaluations: int = Field(default=10, ge=1) def project_path(value: str, project_root: Path) -> Path: @@ -75,7 +75,7 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins "agent_wrapper": { "home_optimizer": { "validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes, - "max_validations": config.max_validations, + "max_evaluations": config.max_evaluations, }, }, }, diff --git a/benchmark/home_eval.py b/benchmark/home_eval.py index 8c4174ed..427ace0c 100644 --- a/benchmark/home_eval.py +++ b/benchmark/home_eval.py @@ -42,7 +42,7 @@ class TestRunConfig(SpecModel): options.debug, options.sampling, options.record_history, - options.max_validations is not None, + options.max_evaluations is not None, options.sampling_policy is not None, options.case_ids is not None, options.query_ids is not None, diff --git a/benchmark/longmemeval/README.md b/benchmark/longmemeval/README.md index e663301c..a589b690 100644 --- a/benchmark/longmemeval/README.md +++ b/benchmark/longmemeval/README.md @@ -141,7 +141,7 @@ The top-level `validation_cycle_time_budget_minutes` configures the host optimiz advisory time between completed formal evaluations. It defaults to 150 minutes; set a positive number to change it or null to disable timed reminders. -The top-level `max_validations` sets the optimizer's formal-evaluation budget +The top-level `max_evaluations` sets the optimizer's formal-evaluation budget (positive integer, default 10). A slot is consumed at first sandbox startup, including later failures or cancellations; initial baseline and debug are excluded. diff --git a/benchmark/longmemeval/README_ZH.md b/benchmark/longmemeval/README_ZH.md index b843179c..ffb8e2da 100644 --- a/benchmark/longmemeval/README_ZH.md +++ b/benchmark/longmemeval/README_ZH.md @@ -128,7 +128,7 @@ Python 中也可调用 `await benchmark.longmemeval.run_home.run(workspace_dir=. 顶层 `validation_cycle_time_budget_minutes` 配置宿主 optimizer 两次已完成正式评测之间的 建议用时,默认 150 分钟;可改为其他正数,或设为 null 关闭时间提醒。 -顶层 `max_validations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。 +顶层 `max_evaluations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。 首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。 准备完成后,用同一份配置进行本地一次性优化: diff --git a/benchmark/longmemeval/config_home.yaml b/benchmark/longmemeval/config_home.yaml index 22991232..764773b5 100644 --- a/benchmark/longmemeval/config_home.yaml +++ b/benchmark/longmemeval/config_home.yaml @@ -3,7 +3,7 @@ # Preserve dev_meta's fixed training split; groups 6-10 remain held out. workspace_dir: benchmark/longmemeval/workspaces/home-groups-1-5 # Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug. -max_validations: 10 +max_evaluations: 10 # Policy defaults for Home evaluation; each call still controls sampling=true/false. sampling_policy: max_cases: 5 diff --git a/benchmark/longmemeval/run_home.py b/benchmark/longmemeval/run_home.py index 04fea91f..f29059e9 100644 --- a/benchmark/longmemeval/run_home.py +++ b/benchmark/longmemeval/run_home.py @@ -34,7 +34,7 @@ class RunHomeConfig(BaseModel): preparation: dict sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy) validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False) - max_validations: int = Field(default=10, ge=1) + max_evaluations: int = Field(default=10, ge=1) def project_path(value: str, project_root: Path) -> Path: @@ -75,7 +75,7 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins "agent_wrapper": { "home_optimizer": { "validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes, - "max_validations": config.max_validations, + "max_evaluations": config.max_evaluations, }, }, }, diff --git a/plugins/home/README.md b/plugins/home/README.md index c98c9a87..f74df8d3 100644 --- a/plugins/home/README.md +++ b/plugins/home/README.md @@ -942,7 +942,7 @@ context tokens) and `components.home.evaluation.default.construction_equivalence_memory_limit_mib` uses the existing OS sandbox memory-limit contract. -During the first `max_validations // 2` non-init formal starts, a fresh +During the first `max_evaluations // 2` non-init formal starts, a fresh `home_candidate_gate` agent reviews each new candidate before sampling or starting an evaluation sandbox. It compares the candidate with its nearest previously evaluated ancestor (falling back to `init`). It accepts runtime mechanism changes @@ -950,7 +950,7 @@ and concrete execution, safety or contract fixes; it rejects prompt/parameter-on tuning, cosmetic changes and dead code during this early exploration phase. Ambiguous evidence is accepted. Baselines, debug, infrastructure retries and later refinement bypass admission. Ordinary unbudgeted `home_evaluate` calls remain -ungated; supplying `max_validations` enables the same admission policy. +ungated; supplying `max_evaluations` enables the same admission policy. The reviewer receives only read-only source exports of the two exact commits and their complete diff under `home/tmp/candidate-gate-*/`. Its Read, Glob, Grep and @@ -1025,8 +1025,11 @@ Evaluation tools queue transitions in request state; the optimizer injects them before the next reasoning call, after all tool results have been written. Thus a transition never separates an API tool call from its matching result. -`components.agent_wrapper.home_optimizer.max_validations` defaults to 10, matching -dev_meta. `get_validation_budget` reports consumed and remaining slots, the search +`components.agent_wrapper.home_optimizer.max_evaluations` defaults to 10, matching +dev_meta. This setting and the `home_evaluate` argument were renamed from +`max_validations`; update existing configuration and callers to `max_evaluations`. +The old name is rejected rather than silently falling back to the default budget. +`get_validation_budget` reports consumed and remaining slots, the search phase and next-candidate guidance. A non-init formal run consumes one slot when its first sandbox initializes successfully, including runs that later fail or are cancelled. Initial baselines, debug, rejected calls and pre-start failures are free. @@ -1034,10 +1037,10 @@ Accounting reuses persistent started-run evidence and survives optimizer restart removing the rebuildable sampling cache does not restore the budget. The evaluator checks the limit while holding the formal-validation lock, before starting another run. At exhaustion it returns `VALIDATION_BUDGET_EXHAUSTED`; the optimizer selects -from existing evidence. The first `max_validations // 2` starts favor structural +from existing evidence. The first `max_evaluations // 2` starts favor structural exploration; later starts favor evidence-driven refinement. This guidance is advisory. Ordinary `home_evaluate` calls remain unbounded unless the host supplies -`max_validations`; all recorded non-init formal starts in that workspace count. +`max_evaluations`; all recorded non-init formal starts in that workspace count. Sessions and model usage follow ReMe's wrapper conventions; each nested call reports its own usage. The optimizer follows dev_meta's compression recovery policy. Recoverable model @@ -1070,7 +1073,7 @@ components: react_config: max_iters: 3000 validation_cycle_time_budget_minutes: 150 - max_validations: 10 + max_evaluations: 10 ``` `evaluation_options` also accepts `base_image`, `fail_fast` and diff --git a/plugins/home/src/reme_home/agent/optimizer.py b/plugins/home/src/reme_home/agent/optimizer.py index efcf3c58..aafc4d02 100644 --- a/plugins/home/src/reme_home/agent/optimizer.py +++ b/plugins/home/src/reme_home/agent/optimizer.py @@ -19,7 +19,7 @@ from ..evaluation.artifacts import contained, read_json, write_json from ..evaluation.component import blocking from ..evaluation.inputs import EvaluationSpec, load_inputs from ..evaluation.history import read_state -from ..evaluation.budget import validate_max_validations, validation_budget_status +from ..evaluation.budget import validate_max_evaluations, validation_budget_status from ..preparation.spec import SpecModel from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory from .diagnostic import diagnostic_tool, evaluation_paths @@ -165,7 +165,10 @@ class OptimizerAgentWrapper(CaseStudyAgentWrapper): def validation_limit(self): """Keep the formal count limit host-owned and independent of elapsed time.""" - return validate_max_validations(self._merged_kwargs({}).get("max_validations", 10)) + options = self._merged_kwargs({}) + if "max_validations" in options: + raise ValueError("Use max_evaluations instead of max_validations") + return validate_max_evaluations(options.get("max_evaluations", 10)) def _create_agent(self, *, options, **kwargs): timer = ( @@ -367,13 +370,13 @@ def verify_final(home, result): return evidence["evaluation_path"] -def validation_budget_tool(home, max_validations): +def validation_budget_tool(home, max_evaluations): """Expose dev_meta's read-only count, phase and next-candidate guidance.""" - validate_max_validations(max_validations) + validate_max_evaluations(max_evaluations) async def get_validation_budget(): """Return started formal validations, remaining count and optimization guidance.""" - return await blocking(validation_budget_status, home, max_validations) + return await blocking(validation_budget_status, home, max_evaluations) return FunctionTool( get_validation_budget, @@ -387,10 +390,10 @@ def validation_budget_tool(home, max_validations): ) -def evaluation_tool(wrapper, options, *, home, timer=None, max_validations=10): +def evaluation_tool(wrapper, options, *, home, timer=None, max_evaluations=10): """Only debug mode is model-selected; the host decides construction replay.""" job = required_job(wrapper, "home_evaluate") - validate_max_validations(max_validations) + validate_max_evaluations(max_evaluations) async def evaluate(debug: bool = False, **kwargs): """Run debug after committing; after debug passes, run a sampled formal evaluation. @@ -408,7 +411,7 @@ def evaluation_tool(wrapper, options, *, home, timer=None, max_validations=10): started_before = await blocking(lambda: set(read_state(home)["started_validations"])) if not debug else set() response = await job( **options, - **({"max_validations": max_validations} if not debug else {}), + **({"max_evaluations": max_evaluations} if not debug else {}), debug=debug, sampling=not debug, record_history=True, @@ -417,7 +420,7 @@ def evaluation_tool(wrapper, options, *, home, timer=None, max_validations=10): reuse_construction_from=None, ) if not debug: - budget = await blocking(validation_budget_status, home, max_validations) + budget = await blocking(validation_budget_status, home, max_evaluations) if isinstance(response.answer, dict): response.answer = {**response.answer, "validation_budget": budget} completed = await blocking(completed_validation, home, response.answer, started_before) @@ -487,8 +490,8 @@ class HomeOptimizeStep(BaseStep): self.context.response.metadata["optimization_report_path"] = report_path try: timer = self.agent_wrapper.create_cycle_timer() - max_validations = self.agent_wrapper.validation_limit() - budget = await blocking(validation_budget_status, home, max_validations) + max_evaluations = self.agent_wrapper.validation_limit() + budget = await blocking(validation_budget_status, home, max_evaluations) reply = await workflow_reply( self.agent_wrapper, home, @@ -501,13 +504,13 @@ class HomeOptimizeStep(BaseStep): tool_factory=lambda backend: [ Edit(backend=backend), diagnostic_tool(self.agent_wrapper), - validation_budget_tool(home, max_validations), + validation_budget_tool(home, max_evaluations), evaluation_tool( self.agent_wrapper, options, home=home, timer=timer, - max_validations=max_validations, + max_evaluations=max_evaluations, ), ], job_names=MUTATION_JOBS, diff --git a/plugins/home/src/reme_home/evaluation/budget.py b/plugins/home/src/reme_home/evaluation/budget.py index 3b4cc752..f39a0c73 100644 --- a/plugins/home/src/reme_home/evaluation/budget.py +++ b/plugins/home/src/reme_home/evaluation/budget.py @@ -3,19 +3,19 @@ from .history import read_state -def validate_max_validations(value): +def validate_max_evaluations(value): """Accept only a positive host-supplied integer, matching dev_meta.""" if not isinstance(value, int) or isinstance(value, bool) or value < 1: - raise ValueError("max_validations must be a positive integer") + raise ValueError("max_evaluations must be a positive integer") return value -def validation_budget_status(home, max_validations): +def validation_budget_status(home, max_evaluations): """Report started formal runs, independently of completion, score or sampler success.""" - validate_max_validations(max_validations) + validate_max_evaluations(max_evaluations) used = sum(not key.startswith("init/") for key in read_state(home)["started_validations"]) - remaining = max(0, max_validations - used) - gated = max_validations // 2 + remaining = max(0, max_evaluations - used) + gated = max_evaluations // 2 if not remaining: phase = "exhausted" guidance = ( @@ -35,7 +35,7 @@ def validation_budget_status(home, max_validations): ) return { "used_validations": used, - "max_validations": max_validations, + "max_evaluations": max_evaluations, "remaining_validations": remaining, "gated_validation_count": gated, "optimization_phase": phase, diff --git a/plugins/home/src/reme_home/evaluation/component.py b/plugins/home/src/reme_home/evaluation/component.py index 10cccd4c..b7a563f8 100644 --- a/plugins/home/src/reme_home/evaluation/component.py +++ b/plugins/home/src/reme_home/evaluation/component.py @@ -91,9 +91,9 @@ class HomeEvaluation(BaseComponent): _validate_directory(self.workspace_path.resolve(), self.workspace.state_path) lock = nullcontext() if spec.debug else _initialization_lock(self.workspace.state_path / "validation.lock") with lock: - budgeted = spec.max_validations is not None and not spec.debug + budgeted = spec.max_evaluations is not None and not spec.debug if budgeted: - budget = await blocking(validation_budget_status, self.workspace.home_path, spec.max_validations) + budget = await blocking(validation_budget_status, self.workspace.home_path, spec.max_evaluations) if budget["remaining_validations"] < 1: return { "status": "rejected", @@ -141,7 +141,7 @@ class HomeEvaluation(BaseComponent): result["validation_budget"] = await blocking( validation_budget_status, self.workspace.home_path, - spec.max_validations, + spec.max_evaluations, ) return result diff --git a/plugins/home/src/reme_home/evaluation/inputs.py b/plugins/home/src/reme_home/evaluation/inputs.py index a319988d..134e73b9 100644 --- a/plugins/home/src/reme_home/evaluation/inputs.py +++ b/plugins/home/src/reme_home/evaluation/inputs.py @@ -29,7 +29,7 @@ class EvaluationSpec(SpecModel): debug: bool = False sampling: bool = False record_history: bool = True - max_validations: int | None = Field(default=None, ge=1) + max_evaluations: int | None = Field(default=None, ge=1) sampling_policy: SamplingPolicy | None = None concurrency: int = Field(default=4, ge=1) timeout_seconds: float = Field(default=7200.0, gt=0, allow_inf_nan=False) @@ -68,7 +68,7 @@ class EvaluationSpec(SpecModel): @model_validator(mode="after") def resolve_mode(self): """Debug stops on failure by default and never reuses scored evidence.""" - if self.max_validations is not None and not self.debug and not self.record_history: + if self.max_evaluations is not None and not self.debug and not self.record_history: raise ValueError("Budgeted formal evaluations require record_history=true") if self.debug and self.reuse_construction_from: raise ValueError("Construction reuse requires training-set") diff --git a/plugins/home/src/reme_home/evaluation/step.py b/plugins/home/src/reme_home/evaluation/step.py index d2dcc4d3..1d71cfea 100644 --- a/plugins/home/src/reme_home/evaluation/step.py +++ b/plugins/home/src/reme_home/evaluation/step.py @@ -17,6 +17,8 @@ class HomeEvaluateStep(BaseStep): if self.context.get("_allowed_paths") is not None: raise ValueError("Home evaluation Jobs require unrestricted access to their workspace output directories") arguments = {name: self.context.get(name) for name in EvaluationSpec.model_fields if name in self.context} + if "max_validations" in self.context: + raise ValueError("Use max_evaluations instead of max_validations") # Do not silently accept a removed toggle that conflicts with reuse paths. if "eval_only" in self.context: raise ValueError("Use reuse_construction_from instead of eval_only") diff --git a/plugins/home/src/reme_home/plugin.yaml b/plugins/home/src/reme_home/plugin.yaml index 65f1c226..cee4cf75 100644 --- a/plugins/home/src/reme_home/plugin.yaml +++ b/plugins/home/src/reme_home/plugin.yaml @@ -97,7 +97,7 @@ application_defaults: max_iters: 60 home_optimizer: backend: home_optimizer_agent - max_validations: 10 + max_evaluations: 10 as_llm: home_optimizer validation_cycle_time_budget_minutes: 150 context_config: @@ -340,7 +340,7 @@ application_defaults: description: "Stop on the first failure; defaults to debug (false for scored runs, true for debug)" max_construction_failure_rate: {type: number, minimum: 0, maximum: 1} max_artifact_bytes: {type: integer, minimum: 1} - max_validations: + max_evaluations: type: [integer, "null"] minimum: 1 description: "Host formal-evaluation budget; early new candidates pass admission review; optimizer injects its limit" diff --git a/plugins/home/tests/test_benchmark_runner.py b/plugins/home/tests/test_benchmark_runner.py index 9a6a92ae..46e6f359 100644 --- a/plugins/home/tests/test_benchmark_runner.py +++ b/plugins/home/tests/test_benchmark_runner.py @@ -89,7 +89,7 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b } assert observed["components"]["agent_wrapper"]["home_optimizer"] == { "validation_cycle_time_budget_minutes": 150, - "max_validations": 10, + "max_evaluations": 10, } application.start.assert_awaited_once() application.close.assert_awaited_once() @@ -156,7 +156,7 @@ async def test_runner_passes_cycle_budget_to_optimizer(tmp_path, monkeypatch, be expected = 150 if budget == "omitted" else budget assert observed["components"]["agent_wrapper"]["home_optimizer"] == { "validation_cycle_time_budget_minutes": expected, - "max_validations": 10, + "max_evaluations": 10, } assert "validation_cycle_time_budget_minutes" not in application.run_job.call_args.kwargs["spec"] application.close.assert_awaited_once() @@ -204,9 +204,9 @@ async def test_runner_formal_budget_configuration(tmp_path, monkeypatch, benchma module = import_module(f"benchmark.{benchmark}.run_home") config = yaml.safe_load(module.DEFAULT_CONFIG.read_text()) if limit == "omitted": - config.pop("max_validations", None) + config.pop("max_evaluations", None) else: - config["max_validations"] = limit + config["max_evaluations"] = limit path = tmp_path / "home.yaml" path.write_text(yaml.safe_dump(config)) application = AsyncMock() @@ -220,16 +220,25 @@ async def test_runner_formal_budget_configuration(tmp_path, monkeypatch, benchma monkeypatch.setattr(module, "Application", factory) if limit in ("omitted", 3): await module.prepare(path) - assert observed["components"]["agent_wrapper"]["home_optimizer"]["max_validations"] == ( + assert observed["components"]["agent_wrapper"]["home_optimizer"]["max_evaluations"] == ( 10 if limit == "omitted" else limit ) - assert "max_validations" not in application.run_job.call_args.kwargs["spec"] + assert "max_evaluations" not in application.run_job.call_args.kwargs["spec"] else: with pytest.raises(ValueError): await module.prepare(path) assert not observed +@pytest.mark.parametrize("module", [beam_runner, lme_runner]) +def test_runner_rejects_old_budget_name(module): + """Reject obsolete config keys instead of using the default budget.""" + config = yaml.safe_load(module.DEFAULT_CONFIG.read_text()) + config["max_validations"] = config.pop("max_evaluations") + with pytest.raises(ValueError): + module.RunHomeConfig.model_validate(config) + + @pytest.mark.asyncio @pytest.mark.parametrize("benchmark", ["beam", "longmemeval"]) @pytest.mark.parametrize("minutes", [30, None]) @@ -237,7 +246,7 @@ async def test_optimizer_lifecycle_uses_same_config_and_does_not_prepare(tmp_pat """A separate optimization launch consumes the same count, timer and sampling settings.""" module = import_module(f"benchmark.{benchmark}.run_home") config = yaml.safe_load(module.DEFAULT_CONFIG.read_text()) - config.update(max_validations=3, validation_cycle_time_budget_minutes=minutes) + config.update(max_evaluations=3, validation_cycle_time_budget_minutes=minutes) path = tmp_path / "home.yaml" path.write_text(yaml.safe_dump(config)) application = AsyncMock() @@ -252,7 +261,7 @@ async def test_optimizer_lifecycle_uses_same_config_and_does_not_prepare(tmp_pat result = await module.optimize(path, workspace_dir=str(tmp_path / "existing"), instruction="Improve retrieval") assert result["status"] == "completed" assert observed["components"]["agent_wrapper"]["home_optimizer"] == { - "max_validations": 3, + "max_evaluations": 3, "validation_cycle_time_budget_minutes": minutes, } application.run_job.assert_awaited_once_with("home_optimize", instruction="Improve retrieval") diff --git a/plugins/home/tests/test_candidate_gate.py b/plugins/home/tests/test_candidate_gate.py index bde00438..96d8da78 100644 --- a/plugins/home/tests/test_candidate_gate.py +++ b/plugins/home/tests/test_candidate_gate.py @@ -75,7 +75,7 @@ async def test_gate_before_sampling_preserves_budget_and_pins_code(app, prepared return original(*args, **kwargs) monkeypatch.setattr(component, "create_sampling_plan", draw) - response = await app.run_job("home_evaluate", max_validations=10, sampling=True) + response = await app.run_job("home_evaluate", max_evaluations=10, sampling=True) report = response.answer["candidate_gate"] assert report["base_commit"] == initial and report["candidate_commit"] == commit saved = read_json(Path(report["review_path"])) @@ -109,16 +109,16 @@ async def test_baseline_debug_unbudgeted_and_late_phase_skip_gate(app, prepared, raise AssertionError("Gate must be skipped") monkeypatch.setattr(candidate_gate, "review_candidate", unexpected) - assert (await app.run_job("home_evaluate", max_validations=10)).success # Baseline. + assert (await app.run_job("home_evaluate", max_evaluations=10)).success # Baseline. commit_candidate(prepared) - assert (await app.run_job("home_evaluate", debug=True, max_validations=10)).success + assert (await app.run_job("home_evaluate", debug=True, max_evaluations=10)).success assert (await app.run_job("home_evaluate")).success # Ordinary explicit evaluations remain ungated. commit_candidate(prepared) - result = await app.run_job("home_evaluate", max_validations=2) # One previous non-init start: second half. + result = await app.run_job("home_evaluate", max_evaluations=2) # One previous non-init start: second half. assert result.success, result.answer assert "candidate_gate" not in result.answer commit_candidate(prepared) - result = await app.run_job("home_evaluate", max_validations=2) + result = await app.run_job("home_evaluate", max_evaluations=2) assert result.answer["error_code"] == "VALIDATION_BUDGET_EXHAUSTED" @@ -132,7 +132,7 @@ async def test_nearest_ancestor_and_infrastructure_retry(app, prepared, sandbox, commit_candidate(prepared) set_model(app, "home_candidate_gate", ScriptedModel(final='{"decision":"accept","reason":"New mechanism"}')) monkeypatch.setattr(FakeFactory, "bad_export", True) - failed = await app.run_job("home_evaluate", max_validations=10, sampling=True) + failed = await app.run_job("home_evaluate", max_evaluations=10, sampling=True) assert not failed.success assert failed.answer["candidate_gate"]["base_commit"] == base @@ -141,9 +141,9 @@ async def test_nearest_ancestor_and_infrastructure_retry(app, prepared, sandbox, monkeypatch.setattr(candidate_gate, "review_candidate", unexpected) monkeypatch.setattr(FakeFactory, "bad_export", False) - recovered = await app.run_job("home_evaluate", max_validations=10, sampling=True) + recovered = await app.run_job("home_evaluate", max_evaluations=10, sampling=True) assert recovered.success, recovered.answer - repeat = await app.run_job("home_evaluate", max_validations=10) + repeat = await app.run_job("home_evaluate", max_evaluations=10) assert repeat.answer["error_code"] == "UNCHANGED_CANDIDATE" @@ -158,7 +158,7 @@ async def test_model_failure_allows_validation_with_saved_diagnostics(app, prepa raise RuntimeError("model unavailable") monkeypatch.setattr(wrapper, "reply", broken) - response = await app.run_job("home_evaluate", max_validations=10) + response = await app.run_job("home_evaluate", max_evaluations=10) assert response.success, response.answer report = response.answer["candidate_gate"] assert report["decision"] == "accept" and report["review_failed"] @@ -177,7 +177,7 @@ async def test_corrected_gate_rejects_before_spending_budget(app, prepared, sand ), ) set_model(app, "home_candidate_gate", model) - response = await app.run_job("home_evaluate", max_validations=10) + response = await app.run_job("home_evaluate", max_evaluations=10) assert not response.success assert response.answer["error_code"] == "CANDIDATE_GATE_REJECTED" report = read_json(Path(response.answer["candidate_gate"]["review_path"])) @@ -199,7 +199,7 @@ async def test_sandbox_setup_failure_never_calls_model_or_evaluates(app, prepare raise RuntimeError("sandbox unavailable") monkeypatch.setattr(workflow_common, "create_case_study_sandbox", unavailable) - result = await app.run_job("home_evaluate", max_validations=10) + result = await app.run_job("home_evaluate", max_evaluations=10) assert not result.success and "sandbox unavailable" in str(result.answer) assert not model.calls and not FakeFactory.instances assert validation_budget_status(prepared, 10)["used_validations"] == 0 @@ -218,9 +218,9 @@ async def test_review_change_race_and_cancellation_release_locks(app, prepared, return {"result": '{"decision":"accept","reason":"Mechanism change"}'} monkeypatch.setattr(wrapper, "reply", paused) - task = asyncio.create_task(app.run_job("home_evaluate", max_validations=10)) + task = asyncio.create_task(app.run_job("home_evaluate", max_evaluations=10)) await asyncio.wait_for(entered.wait(), timeout=10) - concurrent = await app.run_job("home_evaluate", max_validations=10) + concurrent = await app.run_job("home_evaluate", max_evaluations=10) assert not concurrent.success # Formal lock spans review and evaluation. commit_candidate(prepared) release.set() @@ -229,7 +229,7 @@ async def test_review_change_race_and_cancellation_release_locks(app, prepared, assert not FakeFactory.instances entered.clear() release.clear() - task = asyncio.create_task(app.run_job("home_evaluate", max_validations=10)) + task = asyncio.create_task(app.run_job("home_evaluate", max_evaluations=10)) await asyncio.wait_for(entered.wait(), timeout=10) task.cancel() with pytest.raises(asyncio.CancelledError): @@ -237,7 +237,7 @@ async def test_review_change_race_and_cancellation_release_locks(app, prepared, assert not list((prepared / "tmp").glob("candidate-gate-*")) assert validation_budget_status(prepared, 10)["used_validations"] == 0 release.set() - result = await app.run_job("home_evaluate", max_validations=10) + result = await app.run_job("home_evaluate", max_evaluations=10) assert result.success, result.answer diff --git a/plugins/home/tests/test_construction_equivalence.py b/plugins/home/tests/test_construction_equivalence.py index f42636ff..a6052af0 100644 --- a/plugins/home/tests/test_construction_equivalence.py +++ b/plugins/home/tests/test_construction_equivalence.py @@ -129,7 +129,7 @@ async def test_large_resources_export_for_both_reviewers(app, prepared, reviewer ScriptedModel(final=finish), ) force_draw(monkeypatch, 0.1) - response = await app.run_job("home_evaluate", sampling=True, max_validations=10) + response = await app.run_job("home_evaluate", sampling=True, max_evaluations=10) assert response.success, response.answer assert response.answer["execution_mode"] == "query_only_replay" for report in ( @@ -172,7 +172,7 @@ async def test_equivalent_candidate_uses_host_probability( ) set_model(app, "home_construction_equivalence", model) force_draw(monkeypatch, draw) - result = await app.run_job("home_evaluate", sampling=True, max_validations=10) + result = await app.run_job("home_evaluate", sampling=True, max_evaluations=10) assert result.success, result.answer decision = result.answer["automatic_construction_reuse"] assert decision["selected"] == selected and decision["draw"] == draw and decision["probability"] == 0.6 @@ -346,13 +346,13 @@ async def test_infrastructure_retry_preserves_replay_and_selection(app, prepared set_model(app, "home_construction_equivalence", model) force_draw(monkeypatch, 0.1) monkeypatch.setattr(FakeFactory, "bad_export", True) - failed = await app.run_job("home_evaluate", sampling=True, max_validations=10) + failed = await app.run_job("home_evaluate", sampling=True, max_evaluations=10) assert not failed.success failed_manifest = read_json(Path(failed.answer["result_dir"]) / "manifest.json") assert (await app.run_job("home_create_candidate_branch", branch_name="retry-branch")).success monkeypatch.setattr(FakeFactory, "bad_export", False) monkeypatch.setattr(reuse, "select_query_only", lambda: pytest.fail("Recovery must not redraw")) - recovered = await app.run_job("home_evaluate", sampling=True, max_validations=10) + recovered = await app.run_job("home_evaluate", sampling=True, max_evaluations=10) assert recovered.success, recovered.answer manifest = read_json(Path(recovered.answer["result_dir"]) / "manifest.json") assert manifest["automatic_construction_reuse"] == { @@ -411,7 +411,7 @@ async def test_sandbox_failure_stops_before_model_sampling_or_budget(app, prepar raise RuntimeError("sandbox unavailable") monkeypatch.setattr(workflow_common, "create_case_study_sandbox", unavailable) - response = await app.run_job("home_evaluate", sampling=True, max_validations=10) + response = await app.run_job("home_evaluate", sampling=True, max_evaluations=10) assert not response.success and "sandbox unavailable" in str(response.answer) assert not model.calls and len(FakeFactory.instances) == 1 assert not list((prepared / "tmp").glob("construction-equivalence-*")) @@ -459,7 +459,7 @@ async def test_partial_export_and_report_failures_clean_review_files( response = await app.run_job( "home_evaluate", sampling=True, - **({"max_validations": 10} if kind == "candidate-gate" else {}), + **({"max_evaluations": 10} if kind == "candidate-gate" else {}), ) assert not response.success and "failed" in str(response.answer) assert not list((prepared / "tmp").glob(f"{kind}-*")) diff --git a/plugins/home/tests/test_home_eval.py b/plugins/home/tests/test_home_eval.py index f5d53c2b..29092e5d 100644 --- a/plugins/home/tests/test_home_eval.py +++ b/plugins/home/tests/test_home_eval.py @@ -269,7 +269,7 @@ async def test_benchmark_entry_points_select_correct_defaults(benchmark, monkeyp runner.assert_awaited_once_with(module.DEFAULT_CONFIG, expected_benchmark=benchmark, commit="abcdef0") -@pytest.mark.parametrize("options", [{"sampling": True}, {"record_history": True}, {"max_validations": 1}]) +@pytest.mark.parametrize("options", [{"sampling": True}, {"record_history": True}, {"max_evaluations": 1}]) def test_test_runner_rejects_optimizer_modes(options): with pytest.raises(ValueError): home_eval.TestRunConfig.model_validate( diff --git a/plugins/home/tests/test_validation_budget.py b/plugins/home/tests/test_validation_budget.py index 52822f56..2d387ad6 100644 --- a/plugins/home/tests/test_validation_budget.py +++ b/plugins/home/tests/test_validation_budget.py @@ -60,21 +60,39 @@ def test_dev_meta_phase_boundaries(tmp_path, used, phase, remaining): @pytest.mark.parametrize("invalid", [True, False, 0, -1, 1.5, "10", None]) def test_invalid_optimizer_budgets_rejected(invalid): - wrapper = OptimizerAgentWrapper(name="optimizer", max_validations=invalid) + wrapper = OptimizerAgentWrapper(name="optimizer", max_evaluations=invalid) with pytest.raises(ValueError, match="positive integer"): wrapper.validation_limit() if invalid is not None: with pytest.raises(ValueError): - EvaluationSpec(max_validations=invalid) + EvaluationSpec(max_evaluations=invalid) def test_budgeted_evaluation_cannot_disable_started_history(): with pytest.raises(ValueError, match="record_history"): - EvaluationSpec(max_validations=10, record_history=False) - assert EvaluationSpec(debug=True, max_validations=1, record_history=False).debug + EvaluationSpec(max_evaluations=10, record_history=False) + assert EvaluationSpec(debug=True, max_evaluations=1, record_history=False).debug assert OptimizerAgentWrapper(name="optimizer").validation_limit() == 10 +def test_old_budget_name_is_rejected(): + with pytest.raises(ValueError): + EvaluationSpec(max_validations=3) + # Application defaults can supply the new name alongside an old override. + wrapper = OptimizerAgentWrapper(name="optimizer", max_evaluations=10, max_validations=3) + with pytest.raises(ValueError, match="Use max_evaluations"): + wrapper.validation_limit() + + +@pytest.mark.asyncio +async def test_job_rejects_old_budget_before_starting_sandbox(app): + count = len(FakeFactory.instances) + response = await app.run_job("home_evaluate", max_validations=3) + assert not response.success + assert "Use max_evaluations" in str(response.answer) + assert len(FakeFactory.instances) == count + + @pytest.mark.asyncio async def test_budget_tool_reports_persisted_count_without_mutation(tmp_path): seed_history(tmp_path, ["init/base/1", "candidate/hash/1"]) @@ -93,7 +111,7 @@ async def test_exhaustion_blocks_before_pool_and_survives_cache_rebuild(app, pre assert baseline.success assert validation_budget_status(prepared, 1)["used_validations"] == 0 commit_candidate(prepared) - result = await app.run_job("home_evaluate", max_validations=1) + result = await app.run_job("home_evaluate", max_evaluations=1) assert result.success, result.answer assert result.answer["validation_budget"]["remaining_validations"] == 0 count = len(FakeFactory.instances) @@ -101,12 +119,12 @@ async def test_exhaustion_blocks_before_pool_and_survives_cache_rebuild(app, pre (prepared / "sampling/state.json").unlink() assert validation_budget_status(prepared, 1)["used_validations"] == 1 commit_candidate(prepared) - rejected = await app.run_job("home_evaluate", max_validations=1) + rejected = await app.run_job("home_evaluate", max_evaluations=1) assert not rejected.success assert rejected.answer["error_code"] == "VALIDATION_BUDGET_EXHAUSTED" assert rejected.answer["result_dir"] is None assert len(FakeFactory.instances) == count - debug = await app.run_job("home_evaluate", debug=True, max_validations=1) + debug = await app.run_job("home_evaluate", debug=True, max_evaluations=1) assert debug.success, debug.answer assert "validation_budget" not in debug.answer assert validation_budget_status(prepared, 1)["used_validations"] == 1 @@ -129,7 +147,7 @@ async def test_charge_depends_on_first_sandbox_start_not_result(app, prepared, m raise RuntimeError("sandbox initialization failed") monkeypatch.setattr(evaluation, "DockerReMeSandboxFactory", PrestartFailureFactory) - task = asyncio.create_task(app.run_job("home_evaluate", max_validations=1)) + task = asyncio.create_task(app.run_job("home_evaluate", max_evaluations=1)) if outcome == "cancelled": while not FakeFactory.instances: await asyncio.sleep(0) diff --git a/plugins/home/tests/test_workflows.py b/plugins/home/tests/test_workflows.py index 9cfc0ce1..f0af9765 100644 --- a/plugins/home/tests/test_workflows.py +++ b/plugins/home/tests/test_workflows.py @@ -215,7 +215,7 @@ async def test_benchmark_optimizer_applies_budgets_to_real_workflow( { "workspace_dir": str(prepared.parent), "preparation": inputs[0], - "max_validations": 3, + "max_evaluations": 3, "validation_cycle_time_budget_minutes": 30, }, ), @@ -250,7 +250,7 @@ async def test_benchmark_optimizer_applies_budgets_to_real_workflow( assert len(agents) == 1 assert agents[0].validation_cycle_timer.time_budget_minutes == 30 task = "\n".join(msg.get_text_content() or "" for msg in model.calls[0]["messages"]) - assert '"max_validations": 3' in task + assert '"max_evaluations": 3' in task assert '"remaining_validations": 3' in task assert len(FakeFactory.instances) == 1 # Only the earlier baseline ran.