refactor(home): rename max_validations to max_evaluations

This commit is contained in:
酱牛肉 2026-11-02 15:40:35 +08:00
parent eaf38c5da3
commit 75ca3e2011
22 changed files with 120 additions and 85 deletions

View file

@ -168,7 +168,7 @@ The top-level `validation_cycle_time_budget_minutes` configures the host optimiz
advisory time between completed formal evaluations. It defaults to 150 minutes;
set a positive number to change it or null to disable timed reminders.
The top-level `max_validations` sets the optimizer's formal-evaluation budget
The top-level `max_evaluations` sets the optimizer's formal-evaluation budget
(positive integer, default 10). A slot is consumed at first sandbox startup,
including later failures or cancellations; initial baseline and debug are excluded.

View file

@ -157,7 +157,7 @@ Python 中也可调用 `await benchmark.beam.run_home.run(workspace_dir=...)`。
顶层 `validation_cycle_time_budget_minutes` 配置宿主 optimizer 两次已完成正式评测之间的
建议用时,默认 150 分钟;可改为其他正数,或设为 null 关闭时间提醒。
顶层 `max_validations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
顶层 `max_evaluations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。
准备完成后,用同一份配置进行本地一次性优化:

View file

@ -2,7 +2,7 @@
# Paths are relative to the ReMe repository root, not the cwd.
workspace_dir: benchmark/beam/workspaces/home-100K
# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug.
max_validations: 10
max_evaluations: 10
# Target minutes between completed formal optimizer evaluations; null disables timed reminders.
validation_cycle_time_budget_minutes: 150
# Policy defaults for Home evaluation; each call still controls sampling=true/false.

View file

@ -34,7 +34,7 @@ class RunHomeConfig(BaseModel):
preparation: dict
sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False)
max_validations: int = Field(default=10, ge=1)
max_evaluations: int = Field(default=10, ge=1)
def project_path(value: str, project_root: Path) -> Path:
@ -75,7 +75,7 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins
"agent_wrapper": {
"home_optimizer": {
"validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes,
"max_validations": config.max_validations,
"max_evaluations": config.max_evaluations,
},
},
},

View file

@ -42,7 +42,7 @@ class TestRunConfig(SpecModel):
options.debug,
options.sampling,
options.record_history,
options.max_validations is not None,
options.max_evaluations is not None,
options.sampling_policy is not None,
options.case_ids is not None,
options.query_ids is not None,

View file

@ -141,7 +141,7 @@ The top-level `validation_cycle_time_budget_minutes` configures the host optimiz
advisory time between completed formal evaluations. It defaults to 150 minutes;
set a positive number to change it or null to disable timed reminders.
The top-level `max_validations` sets the optimizer's formal-evaluation budget
The top-level `max_evaluations` sets the optimizer's formal-evaluation budget
(positive integer, default 10). A slot is consumed at first sandbox startup,
including later failures or cancellations; initial baseline and debug are excluded.

View file

@ -128,7 +128,7 @@ Python 中也可调用 `await benchmark.longmemeval.run_home.run(workspace_dir=.
顶层 `validation_cycle_time_budget_minutes` 配置宿主 optimizer 两次已完成正式评测之间的
建议用时,默认 150 分钟;可改为其他正数,或设为 null 关闭时间提醒。
顶层 `max_validations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
顶层 `max_evaluations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。
准备完成后,用同一份配置进行本地一次性优化:

View file

@ -3,7 +3,7 @@
# Preserve dev_meta's fixed training split; groups 6-10 remain held out.
workspace_dir: benchmark/longmemeval/workspaces/home-groups-1-5
# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug.
max_validations: 10
max_evaluations: 10
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
sampling_policy:
max_cases: 5

View file

@ -34,7 +34,7 @@ class RunHomeConfig(BaseModel):
preparation: dict
sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False)
max_validations: int = Field(default=10, ge=1)
max_evaluations: int = Field(default=10, ge=1)
def project_path(value: str, project_root: Path) -> Path:
@ -75,7 +75,7 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins
"agent_wrapper": {
"home_optimizer": {
"validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes,
"max_validations": config.max_validations,
"max_evaluations": config.max_evaluations,
},
},
},

View file

@ -942,7 +942,7 @@ context tokens) and
`components.home.evaluation.default.construction_equivalence_memory_limit_mib`
uses the existing OS sandbox memory-limit contract.
During the first `max_validations // 2` non-init formal starts, a fresh
During the first `max_evaluations // 2` non-init formal starts, a fresh
`home_candidate_gate` agent reviews each new candidate before sampling or starting
an evaluation sandbox. It compares the candidate with its nearest previously
evaluated ancestor (falling back to `init`). It accepts runtime mechanism changes
@ -950,7 +950,7 @@ and concrete execution, safety or contract fixes; it rejects prompt/parameter-on
tuning, cosmetic changes and dead code during this early exploration phase.
Ambiguous evidence is accepted. Baselines, debug, infrastructure retries and later
refinement bypass admission. Ordinary unbudgeted `home_evaluate` calls remain
ungated; supplying `max_validations` enables the same admission policy.
ungated; supplying `max_evaluations` enables the same admission policy.
The reviewer receives only read-only source exports of the two exact commits and
their complete diff under `home/tmp/candidate-gate-*/`. Its Read, Glob, Grep and
@ -1025,8 +1025,11 @@ Evaluation tools queue transitions in request state; the optimizer injects them
before the next reasoning call, after all tool results have been written. Thus a
transition never separates an API tool call from its matching result.
`components.agent_wrapper.home_optimizer.max_validations` defaults to 10, matching
dev_meta. `get_validation_budget` reports consumed and remaining slots, the search
`components.agent_wrapper.home_optimizer.max_evaluations` defaults to 10, matching
dev_meta. This setting and the `home_evaluate` argument were renamed from
`max_validations`; update existing configuration and callers to `max_evaluations`.
The old name is rejected rather than silently falling back to the default budget.
`get_validation_budget` reports consumed and remaining slots, the search
phase and next-candidate guidance. A non-init formal run consumes one slot when
its first sandbox initializes successfully, including runs that later fail or are
cancelled. Initial baselines, debug, rejected calls and pre-start failures are free.
@ -1034,10 +1037,10 @@ Accounting reuses persistent started-run evidence and survives optimizer restart
removing the rebuildable sampling cache does not restore the budget. The evaluator
checks the limit while holding the formal-validation lock, before starting another
run. At exhaustion it returns `VALIDATION_BUDGET_EXHAUSTED`; the optimizer selects
from existing evidence. The first `max_validations // 2` starts favor structural
from existing evidence. The first `max_evaluations // 2` starts favor structural
exploration; later starts favor evidence-driven refinement. This guidance is advisory.
Ordinary `home_evaluate` calls remain unbounded unless the host supplies
`max_validations`; all recorded non-init formal starts in that workspace count.
`max_evaluations`; all recorded non-init formal starts in that workspace count.
Sessions and model usage follow ReMe's wrapper conventions; each nested call reports its own usage.
The optimizer follows dev_meta's compression recovery policy. Recoverable model
@ -1070,7 +1073,7 @@ components:
react_config:
max_iters: 3000
validation_cycle_time_budget_minutes: 150
max_validations: 10
max_evaluations: 10
```
`evaluation_options` also accepts `base_image`, `fail_fast` and

View file

@ -19,7 +19,7 @@ from ..evaluation.artifacts import contained, read_json, write_json
from ..evaluation.component import blocking
from ..evaluation.inputs import EvaluationSpec, load_inputs
from ..evaluation.history import read_state
from ..evaluation.budget import validate_max_validations, validation_budget_status
from ..evaluation.budget import validate_max_evaluations, validation_budget_status
from ..preparation.spec import SpecModel
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
from .diagnostic import diagnostic_tool, evaluation_paths
@ -165,7 +165,10 @@ class OptimizerAgentWrapper(CaseStudyAgentWrapper):
def validation_limit(self):
"""Keep the formal count limit host-owned and independent of elapsed time."""
return validate_max_validations(self._merged_kwargs({}).get("max_validations", 10))
options = self._merged_kwargs({})
if "max_validations" in options:
raise ValueError("Use max_evaluations instead of max_validations")
return validate_max_evaluations(options.get("max_evaluations", 10))
def _create_agent(self, *, options, **kwargs):
timer = (
@ -367,13 +370,13 @@ def verify_final(home, result):
return evidence["evaluation_path"]
def validation_budget_tool(home, max_validations):
def validation_budget_tool(home, max_evaluations):
"""Expose dev_meta's read-only count, phase and next-candidate guidance."""
validate_max_validations(max_validations)
validate_max_evaluations(max_evaluations)
async def get_validation_budget():
"""Return started formal validations, remaining count and optimization guidance."""
return await blocking(validation_budget_status, home, max_validations)
return await blocking(validation_budget_status, home, max_evaluations)
return FunctionTool(
get_validation_budget,
@ -387,10 +390,10 @@ def validation_budget_tool(home, max_validations):
)
def evaluation_tool(wrapper, options, *, home, timer=None, max_validations=10):
def evaluation_tool(wrapper, options, *, home, timer=None, max_evaluations=10):
"""Only debug mode is model-selected; the host decides construction replay."""
job = required_job(wrapper, "home_evaluate")
validate_max_validations(max_validations)
validate_max_evaluations(max_evaluations)
async def evaluate(debug: bool = False, **kwargs):
"""Run debug after committing; after debug passes, run a sampled formal evaluation.
@ -408,7 +411,7 @@ def evaluation_tool(wrapper, options, *, home, timer=None, max_validations=10):
started_before = await blocking(lambda: set(read_state(home)["started_validations"])) if not debug else set()
response = await job(
**options,
**({"max_validations": max_validations} if not debug else {}),
**({"max_evaluations": max_evaluations} if not debug else {}),
debug=debug,
sampling=not debug,
record_history=True,
@ -417,7 +420,7 @@ def evaluation_tool(wrapper, options, *, home, timer=None, max_validations=10):
reuse_construction_from=None,
)
if not debug:
budget = await blocking(validation_budget_status, home, max_validations)
budget = await blocking(validation_budget_status, home, max_evaluations)
if isinstance(response.answer, dict):
response.answer = {**response.answer, "validation_budget": budget}
completed = await blocking(completed_validation, home, response.answer, started_before)
@ -487,8 +490,8 @@ class HomeOptimizeStep(BaseStep):
self.context.response.metadata["optimization_report_path"] = report_path
try:
timer = self.agent_wrapper.create_cycle_timer()
max_validations = self.agent_wrapper.validation_limit()
budget = await blocking(validation_budget_status, home, max_validations)
max_evaluations = self.agent_wrapper.validation_limit()
budget = await blocking(validation_budget_status, home, max_evaluations)
reply = await workflow_reply(
self.agent_wrapper,
home,
@ -501,13 +504,13 @@ class HomeOptimizeStep(BaseStep):
tool_factory=lambda backend: [
Edit(backend=backend),
diagnostic_tool(self.agent_wrapper),
validation_budget_tool(home, max_validations),
validation_budget_tool(home, max_evaluations),
evaluation_tool(
self.agent_wrapper,
options,
home=home,
timer=timer,
max_validations=max_validations,
max_evaluations=max_evaluations,
),
],
job_names=MUTATION_JOBS,

View file

@ -3,19 +3,19 @@
from .history import read_state
def validate_max_validations(value):
def validate_max_evaluations(value):
"""Accept only a positive host-supplied integer, matching dev_meta."""
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
raise ValueError("max_validations must be a positive integer")
raise ValueError("max_evaluations must be a positive integer")
return value
def validation_budget_status(home, max_validations):
def validation_budget_status(home, max_evaluations):
"""Report started formal runs, independently of completion, score or sampler success."""
validate_max_validations(max_validations)
validate_max_evaluations(max_evaluations)
used = sum(not key.startswith("init/") for key in read_state(home)["started_validations"])
remaining = max(0, max_validations - used)
gated = max_validations // 2
remaining = max(0, max_evaluations - used)
gated = max_evaluations // 2
if not remaining:
phase = "exhausted"
guidance = (
@ -35,7 +35,7 @@ def validation_budget_status(home, max_validations):
)
return {
"used_validations": used,
"max_validations": max_validations,
"max_evaluations": max_evaluations,
"remaining_validations": remaining,
"gated_validation_count": gated,
"optimization_phase": phase,

View file

@ -91,9 +91,9 @@ class HomeEvaluation(BaseComponent):
_validate_directory(self.workspace_path.resolve(), self.workspace.state_path)
lock = nullcontext() if spec.debug else _initialization_lock(self.workspace.state_path / "validation.lock")
with lock:
budgeted = spec.max_validations is not None and not spec.debug
budgeted = spec.max_evaluations is not None and not spec.debug
if budgeted:
budget = await blocking(validation_budget_status, self.workspace.home_path, spec.max_validations)
budget = await blocking(validation_budget_status, self.workspace.home_path, spec.max_evaluations)
if budget["remaining_validations"] < 1:
return {
"status": "rejected",
@ -141,7 +141,7 @@ class HomeEvaluation(BaseComponent):
result["validation_budget"] = await blocking(
validation_budget_status,
self.workspace.home_path,
spec.max_validations,
spec.max_evaluations,
)
return result

View file

@ -29,7 +29,7 @@ class EvaluationSpec(SpecModel):
debug: bool = False
sampling: bool = False
record_history: bool = True
max_validations: int | None = Field(default=None, ge=1)
max_evaluations: int | None = Field(default=None, ge=1)
sampling_policy: SamplingPolicy | None = None
concurrency: int = Field(default=4, ge=1)
timeout_seconds: float = Field(default=7200.0, gt=0, allow_inf_nan=False)
@ -68,7 +68,7 @@ class EvaluationSpec(SpecModel):
@model_validator(mode="after")
def resolve_mode(self):
"""Debug stops on failure by default and never reuses scored evidence."""
if self.max_validations is not None and not self.debug and not self.record_history:
if self.max_evaluations is not None and not self.debug and not self.record_history:
raise ValueError("Budgeted formal evaluations require record_history=true")
if self.debug and self.reuse_construction_from:
raise ValueError("Construction reuse requires training-set")

View file

@ -17,6 +17,8 @@ class HomeEvaluateStep(BaseStep):
if self.context.get("_allowed_paths") is not None:
raise ValueError("Home evaluation Jobs require unrestricted access to their workspace output directories")
arguments = {name: self.context.get(name) for name in EvaluationSpec.model_fields if name in self.context}
if "max_validations" in self.context:
raise ValueError("Use max_evaluations instead of max_validations")
# Do not silently accept a removed toggle that conflicts with reuse paths.
if "eval_only" in self.context:
raise ValueError("Use reuse_construction_from instead of eval_only")

View file

@ -97,7 +97,7 @@ application_defaults:
max_iters: 60
home_optimizer:
backend: home_optimizer_agent
max_validations: 10
max_evaluations: 10
as_llm: home_optimizer
validation_cycle_time_budget_minutes: 150
context_config:
@ -340,7 +340,7 @@ application_defaults:
description: "Stop on the first failure; defaults to debug (false for scored runs, true for debug)"
max_construction_failure_rate: {type: number, minimum: 0, maximum: 1}
max_artifact_bytes: {type: integer, minimum: 1}
max_validations:
max_evaluations:
type: [integer, "null"]
minimum: 1
description: "Host formal-evaluation budget; early new candidates pass admission review; optimizer injects its limit"

View file

@ -89,7 +89,7 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b
}
assert observed["components"]["agent_wrapper"]["home_optimizer"] == {
"validation_cycle_time_budget_minutes": 150,
"max_validations": 10,
"max_evaluations": 10,
}
application.start.assert_awaited_once()
application.close.assert_awaited_once()
@ -156,7 +156,7 @@ async def test_runner_passes_cycle_budget_to_optimizer(tmp_path, monkeypatch, be
expected = 150 if budget == "omitted" else budget
assert observed["components"]["agent_wrapper"]["home_optimizer"] == {
"validation_cycle_time_budget_minutes": expected,
"max_validations": 10,
"max_evaluations": 10,
}
assert "validation_cycle_time_budget_minutes" not in application.run_job.call_args.kwargs["spec"]
application.close.assert_awaited_once()
@ -204,9 +204,9 @@ async def test_runner_formal_budget_configuration(tmp_path, monkeypatch, benchma
module = import_module(f"benchmark.{benchmark}.run_home")
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
if limit == "omitted":
config.pop("max_validations", None)
config.pop("max_evaluations", None)
else:
config["max_validations"] = limit
config["max_evaluations"] = limit
path = tmp_path / "home.yaml"
path.write_text(yaml.safe_dump(config))
application = AsyncMock()
@ -220,16 +220,25 @@ async def test_runner_formal_budget_configuration(tmp_path, monkeypatch, benchma
monkeypatch.setattr(module, "Application", factory)
if limit in ("omitted", 3):
await module.prepare(path)
assert observed["components"]["agent_wrapper"]["home_optimizer"]["max_validations"] == (
assert observed["components"]["agent_wrapper"]["home_optimizer"]["max_evaluations"] == (
10 if limit == "omitted" else limit
)
assert "max_validations" not in application.run_job.call_args.kwargs["spec"]
assert "max_evaluations" not in application.run_job.call_args.kwargs["spec"]
else:
with pytest.raises(ValueError):
await module.prepare(path)
assert not observed
@pytest.mark.parametrize("module", [beam_runner, lme_runner])
def test_runner_rejects_old_budget_name(module):
"""Reject obsolete config keys instead of using the default budget."""
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
config["max_validations"] = config.pop("max_evaluations")
with pytest.raises(ValueError):
module.RunHomeConfig.model_validate(config)
@pytest.mark.asyncio
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
@pytest.mark.parametrize("minutes", [30, None])
@ -237,7 +246,7 @@ async def test_optimizer_lifecycle_uses_same_config_and_does_not_prepare(tmp_pat
"""A separate optimization launch consumes the same count, timer and sampling settings."""
module = import_module(f"benchmark.{benchmark}.run_home")
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
config.update(max_validations=3, validation_cycle_time_budget_minutes=minutes)
config.update(max_evaluations=3, validation_cycle_time_budget_minutes=minutes)
path = tmp_path / "home.yaml"
path.write_text(yaml.safe_dump(config))
application = AsyncMock()
@ -252,7 +261,7 @@ async def test_optimizer_lifecycle_uses_same_config_and_does_not_prepare(tmp_pat
result = await module.optimize(path, workspace_dir=str(tmp_path / "existing"), instruction="Improve retrieval")
assert result["status"] == "completed"
assert observed["components"]["agent_wrapper"]["home_optimizer"] == {
"max_validations": 3,
"max_evaluations": 3,
"validation_cycle_time_budget_minutes": minutes,
}
application.run_job.assert_awaited_once_with("home_optimize", instruction="Improve retrieval")

View file

@ -75,7 +75,7 @@ async def test_gate_before_sampling_preserves_budget_and_pins_code(app, prepared
return original(*args, **kwargs)
monkeypatch.setattr(component, "create_sampling_plan", draw)
response = await app.run_job("home_evaluate", max_validations=10, sampling=True)
response = await app.run_job("home_evaluate", max_evaluations=10, sampling=True)
report = response.answer["candidate_gate"]
assert report["base_commit"] == initial and report["candidate_commit"] == commit
saved = read_json(Path(report["review_path"]))
@ -109,16 +109,16 @@ async def test_baseline_debug_unbudgeted_and_late_phase_skip_gate(app, prepared,
raise AssertionError("Gate must be skipped")
monkeypatch.setattr(candidate_gate, "review_candidate", unexpected)
assert (await app.run_job("home_evaluate", max_validations=10)).success # Baseline.
assert (await app.run_job("home_evaluate", max_evaluations=10)).success # Baseline.
commit_candidate(prepared)
assert (await app.run_job("home_evaluate", debug=True, max_validations=10)).success
assert (await app.run_job("home_evaluate", debug=True, max_evaluations=10)).success
assert (await app.run_job("home_evaluate")).success # Ordinary explicit evaluations remain ungated.
commit_candidate(prepared)
result = await app.run_job("home_evaluate", max_validations=2) # One previous non-init start: second half.
result = await app.run_job("home_evaluate", max_evaluations=2) # One previous non-init start: second half.
assert result.success, result.answer
assert "candidate_gate" not in result.answer
commit_candidate(prepared)
result = await app.run_job("home_evaluate", max_validations=2)
result = await app.run_job("home_evaluate", max_evaluations=2)
assert result.answer["error_code"] == "VALIDATION_BUDGET_EXHAUSTED"
@ -132,7 +132,7 @@ async def test_nearest_ancestor_and_infrastructure_retry(app, prepared, sandbox,
commit_candidate(prepared)
set_model(app, "home_candidate_gate", ScriptedModel(final='{"decision":"accept","reason":"New mechanism"}'))
monkeypatch.setattr(FakeFactory, "bad_export", True)
failed = await app.run_job("home_evaluate", max_validations=10, sampling=True)
failed = await app.run_job("home_evaluate", max_evaluations=10, sampling=True)
assert not failed.success
assert failed.answer["candidate_gate"]["base_commit"] == base
@ -141,9 +141,9 @@ async def test_nearest_ancestor_and_infrastructure_retry(app, prepared, sandbox,
monkeypatch.setattr(candidate_gate, "review_candidate", unexpected)
monkeypatch.setattr(FakeFactory, "bad_export", False)
recovered = await app.run_job("home_evaluate", max_validations=10, sampling=True)
recovered = await app.run_job("home_evaluate", max_evaluations=10, sampling=True)
assert recovered.success, recovered.answer
repeat = await app.run_job("home_evaluate", max_validations=10)
repeat = await app.run_job("home_evaluate", max_evaluations=10)
assert repeat.answer["error_code"] == "UNCHANGED_CANDIDATE"
@ -158,7 +158,7 @@ async def test_model_failure_allows_validation_with_saved_diagnostics(app, prepa
raise RuntimeError("model unavailable")
monkeypatch.setattr(wrapper, "reply", broken)
response = await app.run_job("home_evaluate", max_validations=10)
response = await app.run_job("home_evaluate", max_evaluations=10)
assert response.success, response.answer
report = response.answer["candidate_gate"]
assert report["decision"] == "accept" and report["review_failed"]
@ -177,7 +177,7 @@ async def test_corrected_gate_rejects_before_spending_budget(app, prepared, sand
),
)
set_model(app, "home_candidate_gate", model)
response = await app.run_job("home_evaluate", max_validations=10)
response = await app.run_job("home_evaluate", max_evaluations=10)
assert not response.success
assert response.answer["error_code"] == "CANDIDATE_GATE_REJECTED"
report = read_json(Path(response.answer["candidate_gate"]["review_path"]))
@ -199,7 +199,7 @@ async def test_sandbox_setup_failure_never_calls_model_or_evaluates(app, prepare
raise RuntimeError("sandbox unavailable")
monkeypatch.setattr(workflow_common, "create_case_study_sandbox", unavailable)
result = await app.run_job("home_evaluate", max_validations=10)
result = await app.run_job("home_evaluate", max_evaluations=10)
assert not result.success and "sandbox unavailable" in str(result.answer)
assert not model.calls and not FakeFactory.instances
assert validation_budget_status(prepared, 10)["used_validations"] == 0
@ -218,9 +218,9 @@ async def test_review_change_race_and_cancellation_release_locks(app, prepared,
return {"result": '{"decision":"accept","reason":"Mechanism change"}'}
monkeypatch.setattr(wrapper, "reply", paused)
task = asyncio.create_task(app.run_job("home_evaluate", max_validations=10))
task = asyncio.create_task(app.run_job("home_evaluate", max_evaluations=10))
await asyncio.wait_for(entered.wait(), timeout=10)
concurrent = await app.run_job("home_evaluate", max_validations=10)
concurrent = await app.run_job("home_evaluate", max_evaluations=10)
assert not concurrent.success # Formal lock spans review and evaluation.
commit_candidate(prepared)
release.set()
@ -229,7 +229,7 @@ async def test_review_change_race_and_cancellation_release_locks(app, prepared,
assert not FakeFactory.instances
entered.clear()
release.clear()
task = asyncio.create_task(app.run_job("home_evaluate", max_validations=10))
task = asyncio.create_task(app.run_job("home_evaluate", max_evaluations=10))
await asyncio.wait_for(entered.wait(), timeout=10)
task.cancel()
with pytest.raises(asyncio.CancelledError):
@ -237,7 +237,7 @@ async def test_review_change_race_and_cancellation_release_locks(app, prepared,
assert not list((prepared / "tmp").glob("candidate-gate-*"))
assert validation_budget_status(prepared, 10)["used_validations"] == 0
release.set()
result = await app.run_job("home_evaluate", max_validations=10)
result = await app.run_job("home_evaluate", max_evaluations=10)
assert result.success, result.answer

View file

@ -129,7 +129,7 @@ async def test_large_resources_export_for_both_reviewers(app, prepared, reviewer
ScriptedModel(final=finish),
)
force_draw(monkeypatch, 0.1)
response = await app.run_job("home_evaluate", sampling=True, max_validations=10)
response = await app.run_job("home_evaluate", sampling=True, max_evaluations=10)
assert response.success, response.answer
assert response.answer["execution_mode"] == "query_only_replay"
for report in (
@ -172,7 +172,7 @@ async def test_equivalent_candidate_uses_host_probability(
)
set_model(app, "home_construction_equivalence", model)
force_draw(monkeypatch, draw)
result = await app.run_job("home_evaluate", sampling=True, max_validations=10)
result = await app.run_job("home_evaluate", sampling=True, max_evaluations=10)
assert result.success, result.answer
decision = result.answer["automatic_construction_reuse"]
assert decision["selected"] == selected and decision["draw"] == draw and decision["probability"] == 0.6
@ -346,13 +346,13 @@ async def test_infrastructure_retry_preserves_replay_and_selection(app, prepared
set_model(app, "home_construction_equivalence", model)
force_draw(monkeypatch, 0.1)
monkeypatch.setattr(FakeFactory, "bad_export", True)
failed = await app.run_job("home_evaluate", sampling=True, max_validations=10)
failed = await app.run_job("home_evaluate", sampling=True, max_evaluations=10)
assert not failed.success
failed_manifest = read_json(Path(failed.answer["result_dir"]) / "manifest.json")
assert (await app.run_job("home_create_candidate_branch", branch_name="retry-branch")).success
monkeypatch.setattr(FakeFactory, "bad_export", False)
monkeypatch.setattr(reuse, "select_query_only", lambda: pytest.fail("Recovery must not redraw"))
recovered = await app.run_job("home_evaluate", sampling=True, max_validations=10)
recovered = await app.run_job("home_evaluate", sampling=True, max_evaluations=10)
assert recovered.success, recovered.answer
manifest = read_json(Path(recovered.answer["result_dir"]) / "manifest.json")
assert manifest["automatic_construction_reuse"] == {
@ -411,7 +411,7 @@ async def test_sandbox_failure_stops_before_model_sampling_or_budget(app, prepar
raise RuntimeError("sandbox unavailable")
monkeypatch.setattr(workflow_common, "create_case_study_sandbox", unavailable)
response = await app.run_job("home_evaluate", sampling=True, max_validations=10)
response = await app.run_job("home_evaluate", sampling=True, max_evaluations=10)
assert not response.success and "sandbox unavailable" in str(response.answer)
assert not model.calls and len(FakeFactory.instances) == 1
assert not list((prepared / "tmp").glob("construction-equivalence-*"))
@ -459,7 +459,7 @@ async def test_partial_export_and_report_failures_clean_review_files(
response = await app.run_job(
"home_evaluate",
sampling=True,
**({"max_validations": 10} if kind == "candidate-gate" else {}),
**({"max_evaluations": 10} if kind == "candidate-gate" else {}),
)
assert not response.success and "failed" in str(response.answer)
assert not list((prepared / "tmp").glob(f"{kind}-*"))

View file

@ -269,7 +269,7 @@ async def test_benchmark_entry_points_select_correct_defaults(benchmark, monkeyp
runner.assert_awaited_once_with(module.DEFAULT_CONFIG, expected_benchmark=benchmark, commit="abcdef0")
@pytest.mark.parametrize("options", [{"sampling": True}, {"record_history": True}, {"max_validations": 1}])
@pytest.mark.parametrize("options", [{"sampling": True}, {"record_history": True}, {"max_evaluations": 1}])
def test_test_runner_rejects_optimizer_modes(options):
with pytest.raises(ValueError):
home_eval.TestRunConfig.model_validate(

View file

@ -60,21 +60,39 @@ def test_dev_meta_phase_boundaries(tmp_path, used, phase, remaining):
@pytest.mark.parametrize("invalid", [True, False, 0, -1, 1.5, "10", None])
def test_invalid_optimizer_budgets_rejected(invalid):
wrapper = OptimizerAgentWrapper(name="optimizer", max_validations=invalid)
wrapper = OptimizerAgentWrapper(name="optimizer", max_evaluations=invalid)
with pytest.raises(ValueError, match="positive integer"):
wrapper.validation_limit()
if invalid is not None:
with pytest.raises(ValueError):
EvaluationSpec(max_validations=invalid)
EvaluationSpec(max_evaluations=invalid)
def test_budgeted_evaluation_cannot_disable_started_history():
with pytest.raises(ValueError, match="record_history"):
EvaluationSpec(max_validations=10, record_history=False)
assert EvaluationSpec(debug=True, max_validations=1, record_history=False).debug
EvaluationSpec(max_evaluations=10, record_history=False)
assert EvaluationSpec(debug=True, max_evaluations=1, record_history=False).debug
assert OptimizerAgentWrapper(name="optimizer").validation_limit() == 10
def test_old_budget_name_is_rejected():
with pytest.raises(ValueError):
EvaluationSpec(max_validations=3)
# Application defaults can supply the new name alongside an old override.
wrapper = OptimizerAgentWrapper(name="optimizer", max_evaluations=10, max_validations=3)
with pytest.raises(ValueError, match="Use max_evaluations"):
wrapper.validation_limit()
@pytest.mark.asyncio
async def test_job_rejects_old_budget_before_starting_sandbox(app):
count = len(FakeFactory.instances)
response = await app.run_job("home_evaluate", max_validations=3)
assert not response.success
assert "Use max_evaluations" in str(response.answer)
assert len(FakeFactory.instances) == count
@pytest.mark.asyncio
async def test_budget_tool_reports_persisted_count_without_mutation(tmp_path):
seed_history(tmp_path, ["init/base/1", "candidate/hash/1"])
@ -93,7 +111,7 @@ async def test_exhaustion_blocks_before_pool_and_survives_cache_rebuild(app, pre
assert baseline.success
assert validation_budget_status(prepared, 1)["used_validations"] == 0
commit_candidate(prepared)
result = await app.run_job("home_evaluate", max_validations=1)
result = await app.run_job("home_evaluate", max_evaluations=1)
assert result.success, result.answer
assert result.answer["validation_budget"]["remaining_validations"] == 0
count = len(FakeFactory.instances)
@ -101,12 +119,12 @@ async def test_exhaustion_blocks_before_pool_and_survives_cache_rebuild(app, pre
(prepared / "sampling/state.json").unlink()
assert validation_budget_status(prepared, 1)["used_validations"] == 1
commit_candidate(prepared)
rejected = await app.run_job("home_evaluate", max_validations=1)
rejected = await app.run_job("home_evaluate", max_evaluations=1)
assert not rejected.success
assert rejected.answer["error_code"] == "VALIDATION_BUDGET_EXHAUSTED"
assert rejected.answer["result_dir"] is None
assert len(FakeFactory.instances) == count
debug = await app.run_job("home_evaluate", debug=True, max_validations=1)
debug = await app.run_job("home_evaluate", debug=True, max_evaluations=1)
assert debug.success, debug.answer
assert "validation_budget" not in debug.answer
assert validation_budget_status(prepared, 1)["used_validations"] == 1
@ -129,7 +147,7 @@ async def test_charge_depends_on_first_sandbox_start_not_result(app, prepared, m
raise RuntimeError("sandbox initialization failed")
monkeypatch.setattr(evaluation, "DockerReMeSandboxFactory", PrestartFailureFactory)
task = asyncio.create_task(app.run_job("home_evaluate", max_validations=1))
task = asyncio.create_task(app.run_job("home_evaluate", max_evaluations=1))
if outcome == "cancelled":
while not FakeFactory.instances:
await asyncio.sleep(0)

View file

@ -215,7 +215,7 @@ async def test_benchmark_optimizer_applies_budgets_to_real_workflow(
{
"workspace_dir": str(prepared.parent),
"preparation": inputs[0],
"max_validations": 3,
"max_evaluations": 3,
"validation_cycle_time_budget_minutes": 30,
},
),
@ -250,7 +250,7 @@ async def test_benchmark_optimizer_applies_budgets_to_real_workflow(
assert len(agents) == 1
assert agents[0].validation_cycle_timer.time_budget_minutes == 30
task = "\n".join(msg.get_text_content() or "" for msg in model.calls[0]["messages"])
assert '"max_validations": 3' in task
assert '"max_evaluations": 3' in task
assert '"remaining_validations": 3' in task
assert len(FakeFactory.instances) == 1 # Only the earlier baseline ran.